diff --git a/streampipes-client-api/pom.xml b/streampipes-client-api/pom.xml index d54bbbb9a8..d3f9835646 100644 --- a/streampipes-client-api/pom.xml +++ b/streampipes-client-api/pom.xml @@ -45,6 +45,11 @@ streampipes-model 0.93.0-SNAPSHOT + + org.apache.streampipes + streampipes-service-discovery-api + 0.93.0-SNAPSHOT + org.apache.httpcomponents fluent-hc diff --git a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java index d8a4ee5916..1d6042d59b 100644 --- a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java +++ b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IAdminApi.java @@ -18,13 +18,24 @@ package org.apache.streampipes.client.api; -import org.apache.streampipes.model.config.MessagingSettings; +import org.apache.streampipes.model.configuration.MessagingSettings; import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; import org.apache.streampipes.model.function.FunctionDefinition; import java.util.List; public interface IAdminApi { + + void registerService(SpServiceRegistration serviceRegistration); + + void deregisterService(String serviceId); + + void registerServiceConfiguration(SpServiceConfiguration serviceConfiguration); + + SpServiceConfiguration getServiceConfiguration(String serviceGroup); + void registerAdapters(List adapters); void registerFunctions(List functions); diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AbstractClientApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AbstractClientApi.java index 2297b5302f..c4df217fee 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AbstractClientApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AbstractClientApi.java @@ -20,14 +20,18 @@ import org.apache.streampipes.client.http.DeleteRequest; import org.apache.streampipes.client.http.GetRequest; import org.apache.streampipes.client.http.PostRequestWithPayloadResponse; +import org.apache.streampipes.client.http.PostRequestWithoutPayload; import org.apache.streampipes.client.http.PostRequestWithoutPayloadResponse; import org.apache.streampipes.client.http.PutRequest; import org.apache.streampipes.client.model.StreamPipesClientConfig; import org.apache.streampipes.client.serializer.ObjectSerializer; import org.apache.streampipes.client.serializer.Serializer; import org.apache.streampipes.client.util.StreamPipesApiPath; +import org.apache.streampipes.commons.exceptions.SpHttpErrorStatusCode; import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import java.util.Optional; + public class AbstractClientApi { protected StreamPipesClientConfig clientConfig; @@ -41,6 +45,11 @@ protected T post(StreamPipesApiPath apiPath, Class responseClass) { return new PostRequestWithPayloadResponse<>(clientConfig, apiPath, serializer, responseClass).executeRequest(); } + protected void post(StreamPipesApiPath apiPath) { + ObjectSerializer serializer = new ObjectSerializer<>(); + new PostRequestWithoutPayload<>(clientConfig, apiPath, serializer).executeRequest(); + } + protected void post(StreamPipesApiPath apiPath, T object) { ObjectSerializer serializer = new ObjectSerializer<>(); new PostRequestWithoutPayloadResponse<>(clientConfig, apiPath, serializer, object).executeRequest(); @@ -73,4 +82,18 @@ protected T getSingle(StreamPipesApiPath apiPath, Class targetClass) thro ObjectSerializer serializer = new ObjectSerializer<>(); return new GetRequest<>(clientConfig, apiPath, targetClass, serializer).executeRequest(); } + + protected Optional getSingleOpt(StreamPipesApiPath apiPath, + Class targetClass) throws SpRuntimeException { + try { + ObjectSerializer serializer = new ObjectSerializer<>(); + return Optional.of(new GetRequest<>(clientConfig, apiPath, targetClass, serializer).executeRequest()); + } catch (SpHttpErrorStatusCode e) { + if (e.getHttpStatusCode() == 404) { + return Optional.empty(); + } else { + throw e; + } + } + } } diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java index 74c89fbef8..3f9403614b 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/AdminApi.java @@ -19,8 +19,10 @@ import org.apache.streampipes.client.model.StreamPipesClientConfig; import org.apache.streampipes.client.util.StreamPipesApiPath; -import org.apache.streampipes.model.config.MessagingSettings; +import org.apache.streampipes.model.configuration.MessagingSettings; import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; import org.apache.streampipes.model.function.FunctionDefinition; import org.apache.streampipes.model.message.SuccessMessage; @@ -32,6 +34,29 @@ public AdminApi(StreamPipesClientConfig clientConfig) { super(clientConfig); } + @Override + public void registerService(SpServiceRegistration serviceRegistration) { + post(getExtensionsServiceRegistrationPath(), serviceRegistration); + } + + @Override + public void deregisterService(String serviceId) { + post(getExtensionsServiceRegistrationPath().addToPath(serviceId)); + } + + @Override + public void registerServiceConfiguration(SpServiceConfiguration serviceConfiguration) { + post(getExtensionsServiceConfigurationPath(), serviceConfiguration); + } + + @Override + public SpServiceConfiguration getServiceConfiguration(String serviceGroup) { + var opt = getSingleOpt( + getExtensionsServiceConfigurationPath().addToPath(serviceGroup), SpServiceConfiguration.class); + + return opt.orElseGet(SpServiceConfiguration::new); + } + @Override public void registerAdapters(List adapters) { post(getConnectPath(), adapters); @@ -52,10 +77,21 @@ public MessagingSettings getMessagingSettings() { return getSingle(getMessagingSettingsPath(), MessagingSettings.class); } + private StreamPipesApiPath getExtensionsServiceRegistrationPath() { + return StreamPipesApiPath + .fromBaseApiPath() + .addToPath("extensions-services"); + } + + private StreamPipesApiPath getExtensionsServiceConfigurationPath() { + return StreamPipesApiPath + .fromBaseApiPath() + .addToPath("extensions-services-configurations"); + } + private StreamPipesApiPath getMessagingSettingsPath() { return StreamPipesApiPath .fromBaseApiPath() - .addToPath("consul") .addToPath("messaging"); } diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/http/HttpRequest.java b/streampipes-client/src/main/java/org/apache/streampipes/client/http/HttpRequest.java index 8edea1a4a4..c11275c285 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/http/HttpRequest.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/http/HttpRequest.java @@ -98,19 +98,17 @@ public T executeRequest() { try { HttpResponse response = request.execute().returnResponse(); StatusLine status = response.getStatusLine(); - if (status.getStatusCode() == HttpStatus.SC_OK) { + if (status.getStatusCode() == HttpStatus.SC_OK || status.getStatusCode() == HttpStatus.SC_CREATED) { return afterRequest(serializer, response.getEntity()); } else { switch (status.getStatusCode()) { - case HttpStatus.SC_UNAUTHORIZED: - throw new SpHttpErrorStatusCode( - " 401 - Access to this resource is forbidden - did you provide a poper API key or client secret?", - 401); - case HttpStatus.SC_NOT_FOUND: - throw new SpHttpErrorStatusCode(" 404 - The requested resource could not be found.", 404); - default: - throw new SpHttpErrorStatusCode(status.getStatusCode() + " - " + status.getReasonPhrase(), - status.getStatusCode()); + case HttpStatus.SC_UNAUTHORIZED -> throw new SpHttpErrorStatusCode( + " 401 - Access to this resource is forbidden - did you provide a poper API key or client secret?", + 401); + case HttpStatus.SC_NOT_FOUND -> + throw new SpHttpErrorStatusCode(" 404 - The requested resource could not be found.", 404); + default -> throw new SpHttpErrorStatusCode(status.getStatusCode() + " - " + status.getReasonPhrase(), + status.getStatusCode()); } } } catch (IOException e) { diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/http/PostRequest.java b/streampipes-client/src/main/java/org/apache/streampipes/client/http/PostRequest.java index fc6de13742..15293e1b56 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/http/PostRequest.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/http/PostRequest.java @@ -27,7 +27,7 @@ public abstract class PostRequest extends HttpRequest { private K body; - private boolean withBody; + private final boolean withBody; public PostRequest(StreamPipesClientConfig clientConfig, StreamPipesApiPath apiPath, diff --git a/ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.css b/streampipes-client/src/main/java/org/apache/streampipes/client/http/PostRequestWithoutPayload.java similarity index 54% rename from ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.css rename to streampipes-client/src/main/java/org/apache/streampipes/client/http/PostRequestWithoutPayload.java index 2b4c9a524b..d685e21695 100644 --- a/ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.css +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/http/PostRequestWithoutPayload.java @@ -16,44 +16,23 @@ * */ -table { - width: 100%; -} +package org.apache.streampipes.client.http; -.service-icon-passing { - cursor: default; - color: #4caf50; -} +import org.apache.streampipes.client.model.StreamPipesClientConfig; +import org.apache.streampipes.client.serializer.Serializer; +import org.apache.streampipes.client.util.StreamPipesApiPath; -.service-icon-critical { - cursor: default; - color: #f44336; -} +import org.apache.http.HttpEntity; -tr.consul-configuration-detail-row { - height: 0; -} +public class PostRequestWithoutPayload extends PostRequest { -tr.consul-configuration-row:not(.consul-configuration-row-expanded):hover { - background: var(--color-bg-2); -} - -tr.consul-configuration-row:not(.consul-configuration-row-expanded):active { - background: var(--color-bg-1); -} - -.consul-configuration-row td { - border-bottom-width: 0; -} - -.consul-configuration-detail { - overflow: hidden; - display: flex; -} - -.mat-table { -} + public PostRequestWithoutPayload(StreamPipesClientConfig clientConfig, + StreamPipesApiPath apiPath, Serializer serializer) { + super(clientConfig, apiPath, serializer); + } -.consul-service { - padding: 5px; + @Override + protected Void afterRequest(Serializer serializer, HttpEntity entity) { + return null; + } } diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java index 23f16489ff..a56b1257c4 100644 --- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java @@ -32,8 +32,14 @@ public enum Envs { SP_CONSUL_HOST("SP_CONSUL_HOST", DefaultEnvValues.CONSUL_HOST_DEFAULT, DefaultEnvValues.LOCALHOST), + + SP_CORE_SCHEME("SP_CORE_SCHEME", "http", "http"), + SP_CORE_HOST("SP_CORE_HOST", "backend", "localhost"), + SP_CORE_PORT("SP_CORE_PORT", "8030", "8030"), + SP_CONSUL_PORT("SP_CONSUL_PORT", DefaultEnvValues.CONSUL_PORT_DEFAULT), SP_KAFKA_RETENTION_MS("SP_KAFKA_RETENTION_MS", DefaultEnvValues.SP_KAFKA_RETENTION_MS_DEFAULT), + SP_PRIORITIZED_PROTOCOL("SP_PRIORITIZED_PROTOCOL", "kafka"), SP_JWT_SECRET("JWT_SECRET"), SP_JWT_SIGNING_MODE("SP_JWT_SIGNING_MODE"), SP_JWT_PRIVATE_KEY_LOC("SP_JWT_PRIVATE_KEY_LOC"), diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java index 4f6d71cb1f..546d42d4b1 100644 --- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java @@ -50,6 +50,21 @@ public IntEnvironmentVariable getServicePort() { return new IntEnvironmentVariable(Envs.SP_PORT); } + @Override + public StringEnvironmentVariable getSpCoreScheme() { + return new StringEnvironmentVariable(Envs.SP_CORE_SCHEME); + } + + @Override + public StringEnvironmentVariable getSpCoreHost() { + return new StringEnvironmentVariable(Envs.SP_CORE_HOST); + } + + @Override + public IntEnvironmentVariable getSpCorePort() { + return new IntEnvironmentVariable(Envs.SP_CORE_PORT); + } + @Override public StringEnvironmentVariable getTsStorageProtocol() { return new StringEnvironmentVariable(Envs.SP_TS_STORAGE_PROTOCOL); @@ -149,6 +164,11 @@ public StringEnvironmentVariable getKafkaRetentionTimeMs() { return new StringEnvironmentVariable(Envs.SP_KAFKA_RETENTION_MS); } + @Override + public StringEnvironmentVariable getPrioritizedProtocol() { + return new StringEnvironmentVariable(Envs.SP_PRIORITIZED_PROTOCOL); + } + @Override public BooleanEnvironmentVariable getSetupInstallPipelineElements() { return new BooleanEnvironmentVariable(Envs.SP_SETUP_INSTALL_PIPELINE_ELEMENTS); diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java index 7217d261af..67e18c6535 100644 --- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java @@ -40,6 +40,9 @@ public interface Environment { IntEnvironmentVariable getServicePort(); + StringEnvironmentVariable getSpCoreScheme(); + StringEnvironmentVariable getSpCoreHost(); + IntEnvironmentVariable getSpCorePort(); // Time series storage env variables @@ -89,6 +92,8 @@ public interface Environment { // Messaging StringEnvironmentVariable getKafkaRetentionTimeMs(); + StringEnvironmentVariable getPrioritizedProtocol(); + // Setup BooleanEnvironmentVariable getSetupInstallPipelineElements(); diff --git a/streampipes-config/pom.xml b/streampipes-config/pom.xml index 6f70e62e59..562dd79289 100644 --- a/streampipes-config/pom.xml +++ b/streampipes-config/pom.xml @@ -34,6 +34,11 @@ streampipes-model 0.93.0-SNAPSHOT + + org.apache.streampipes + streampipes-model-client + 0.93.0-SNAPSHOT + org.apache.streampipes streampipes-serializers-json diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfig.java b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfig.java index d626021c31..f07366814b 100644 --- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfig.java +++ b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfig.java @@ -19,84 +19,27 @@ package org.apache.streampipes.config.backend; -import org.apache.streampipes.commons.environment.Environment; import org.apache.streampipes.commons.environment.Environments; import org.apache.streampipes.commons.random.TokenGenerator; -import org.apache.streampipes.config.backend.model.EmailConfig; -import org.apache.streampipes.config.backend.model.GeneralConfig; -import org.apache.streampipes.config.backend.model.LocalAuthConfig; -import org.apache.streampipes.model.config.MessagingSettings; +import org.apache.streampipes.model.configuration.DefaultMessagingSettings; +import org.apache.streampipes.model.configuration.EmailConfig; +import org.apache.streampipes.model.configuration.GeneralConfig; +import org.apache.streampipes.model.configuration.LocalAuthConfig; +import org.apache.streampipes.model.configuration.MessagingSettings; import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; import org.apache.streampipes.svcdiscovery.api.SpConfig; -import java.io.File; - public enum BackendConfig { INSTANCE; - private final char[] possibleCharacters = - ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~`!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?") - .toCharArray(); - private SpConfig config; + private final SpConfig config; BackendConfig() { config = SpServiceDiscovery.getSpConfig("backend"); - - config.register(BackendConfigKeys.SERVICE_NAME, "Backend", "Backend Configuration"); - - config.register(BackendConfigKeys.BACKEND_HOST, "backend", "Hostname for backend"); - config.register(BackendConfigKeys.BACKEND_PORT, 8030, "Port for backend"); - - config.register(BackendConfigKeys.JMS_HOST, "activemq", "Hostname for backend service for active mq"); - config.register(BackendConfigKeys.JMS_PORT, 61616, "Port for backend service for active mq"); - config.register(BackendConfigKeys.MQTT_HOST, "activemq", "Hostname of mqtt service"); - config.register(BackendConfigKeys.MQTT_PORT, 1883, "Port of mqtt service"); - config.register(BackendConfigKeys.NATS_HOST, "nats", "Hostname of Nats"); - config.register(BackendConfigKeys.NATS_PORT, 4222, "Port of Nats"); - config.register(BackendConfigKeys.KAFKA_HOST, "kafka", "Hostname for backend service for kafka"); - config.register(BackendConfigKeys.KAFKA_PORT, 9092, "Port for backend service for kafka"); - config.register(BackendConfigKeys.PULSAR_URL, "pulsar://localhost:6650", "Apache Pulsar URL"); - config.register(BackendConfigKeys.ZOOKEEPER_HOST, "zookeeper", "Hostname for backend service for zookeeper"); - config.register(BackendConfigKeys.ZOOKEEPER_PORT, 2181, "Port for backend service for zookeeper"); config.register(BackendConfigKeys.IS_CONFIGURED, false, "Boolean that indicates whether streampipes is " + "already configured or not"); config.register(BackendConfigKeys.IS_SETUP_RUNNING, false, "Boolean that indicates whether the initial setup " + "is currently running"); - config.register(BackendConfigKeys.ASSETS_DIR, makeAssetLocation(), - "The directory where " + "pipeline element assets are stored."); - config.register(BackendConfigKeys.FILES_DIR, makeFileLocation(), - "The directory where " + "pipeline element files are stored."); - config.registerObject(BackendConfigKeys.MESSAGING_SETTINGS, DefaultMessagingSettings.make(), - "Default Messaging Settings"); - - config.registerObject(BackendConfigKeys.LOCAL_AUTH_CONFIG, LocalAuthConfig.fromDefaults(getJwtSecret()), - "Local authentication settings"); - config.registerObject(BackendConfigKeys.EMAIL_CONFIG, EmailConfig.fromDefaults(), "Email settings"); - config.registerObject(BackendConfigKeys.GENERAL_CONFIG, new GeneralConfig(), "General settings"); - } - - private String makeAssetLocation() { - return makeStreamPipesHomeLocation() - + "assets"; - } - - private String makeFileLocation() { - return makeStreamPipesHomeLocation() - + "files"; - } - - private String makeStreamPipesHomeLocation() { - var userDefinedAssetDir = getEnvironment().getCoreAssetBaseDir(); - var assetDirAppendix = getSpAssetDirAppendix(); - if (userDefinedAssetDir.exists()) { - return userDefinedAssetDir.getValue() + assetDirAppendix; - } else { - return System.getProperty("user.home") + assetDirAppendix; - } - } - - private String getSpAssetDirAppendix() { - return File.separator + ".streampipes" + File.separator; } public String getJmsHost() { @@ -145,21 +88,13 @@ public int getZookeeperPort() { public MessagingSettings getMessagingSettings() { return config.getObject(BackendConfigKeys.MESSAGING_SETTINGS, MessagingSettings.class, - DefaultMessagingSettings.make()); + new DefaultMessagingSettings().make()); } public boolean isConfigured() { return config.getBoolean(BackendConfigKeys.IS_CONFIGURED); } - public void setKafkaHost(String s) { - config.setString(BackendConfigKeys.KAFKA_HOST, s); - } - - public void setMessagingSettings(MessagingSettings settings) { - config.setObject(BackendConfigKeys.MESSAGING_SETTINGS, settings); - } - public void setIsConfigured(boolean b) { config.setBoolean(BackendConfigKeys.IS_CONFIGURED, b); } @@ -181,28 +116,12 @@ public EmailConfig getEmailConfig() { return config.getObject(BackendConfigKeys.EMAIL_CONFIG, EmailConfig.class, EmailConfig.fromDefaults()); } - public void updateEmailConfig(EmailConfig emailConfig) { - config.setObject(BackendConfigKeys.EMAIL_CONFIG, emailConfig); - } - public GeneralConfig getGeneralConfig() { return config.getObject(BackendConfigKeys.GENERAL_CONFIG, GeneralConfig.class, new GeneralConfig()); } - public void updateGeneralConfig(GeneralConfig generalConfig) { - config.setObject(BackendConfigKeys.GENERAL_CONFIG, generalConfig); - } - - public void updateLocalAuthConfig(LocalAuthConfig authConfig) { - config.setObject(BackendConfigKeys.LOCAL_AUTH_CONFIG, authConfig); - } - - public void updateSetupStatus(boolean status) { - config.setBoolean(BackendConfigKeys.IS_SETUP_RUNNING, status); - } - private String getJwtSecret() { - var env = getEnvironment(); + var env = Environments.getEnvironment(); return env.getJwtSecret().getValueOrResolve(this::makeDefaultJwtSecret); } @@ -210,8 +129,7 @@ private String makeDefaultJwtSecret() { return TokenGenerator.generateNewToken(); } - private Environment getEnvironment() { - return Environments.getEnvironment(); + public void updateSetupStatus(boolean status) { + config.setBoolean(BackendConfigKeys.IS_SETUP_RUNNING, status); } - } diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfigKeys.java b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfigKeys.java index 0fe2db1525..b0268f46c6 100644 --- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfigKeys.java +++ b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/BackendConfigKeys.java @@ -41,9 +41,4 @@ public class BackendConfigKeys { public static final String LOCAL_AUTH_CONFIG = "SP_LOCAL_AUTH_CONFIG"; public static final String EMAIL_CONFIG = "SP_EMAIL_CONFIG"; public static final String GENERAL_CONFIG = "SP_GENERAL_CONFIG"; - - - public static final String SERVICE_NAME = "SP_SERVICE_NAME"; - - public static final String PRIORITIZED_PROTOCOL = "SP_PRIORITIZED_PROTOCOL"; } diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/DefaultMessagingSettings.java b/streampipes-config/src/main/java/org/apache/streampipes/config/backend/DefaultMessagingSettings.java deleted file mode 100644 index fa4e5ed8f3..0000000000 --- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/DefaultMessagingSettings.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.config.backend; - -import org.apache.streampipes.model.config.MessagingSettings; -import org.apache.streampipes.model.config.SpDataFormat; -import org.apache.streampipes.model.config.SpProtocol; - -import java.util.Arrays; -import java.util.List; - -public class DefaultMessagingSettings { - - public static MessagingSettings make() { - List protocolList; - if (System.getenv(BackendConfigKeys.PRIORITIZED_PROTOCOL) != null) { - switch (System.getenv(BackendConfigKeys.PRIORITIZED_PROTOCOL).toLowerCase()) { - case "mqtt": - protocolList = - Arrays.asList(SpProtocol.MQTT, SpProtocol.KAFKA, SpProtocol.JMS, SpProtocol.NATS, SpProtocol.PULSAR); - break; - case "kafka": - protocolList = - Arrays.asList(SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS, SpProtocol.PULSAR); - break; - case "jms": - protocolList = - Arrays.asList(SpProtocol.JMS, SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.NATS, SpProtocol.PULSAR); - break; - case "nats": - protocolList = - Arrays.asList(SpProtocol.NATS, SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.PULSAR); - break; - case "pulsar": - protocolList = - Arrays.asList(SpProtocol.PULSAR, SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS); - break; - default: - protocolList = - Arrays.asList(SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS, SpProtocol.PULSAR); - } - } else { - protocolList = - Arrays.asList(SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS, SpProtocol.PULSAR); - } - - return new MessagingSettings( - 1638400, 5000012, 20, 2, - Arrays.asList(SpDataFormat.JSON, SpDataFormat.CBOR, SpDataFormat.FST, SpDataFormat.SMILE), - protocolList); - } -} diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java index 6d8dd7c305..74c771358e 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java @@ -18,8 +18,7 @@ package org.apache.streampipes.connect.management.util; -import org.apache.streampipes.config.backend.BackendConfig; -import org.apache.streampipes.model.config.SpProtocol; +import org.apache.streampipes.model.configuration.SpProtocol; import org.apache.streampipes.model.grounding.EventGrounding; import org.apache.streampipes.model.grounding.JmsTransportProtocol; import org.apache.streampipes.model.grounding.KafkaTransportProtocol; @@ -39,41 +38,45 @@ public class GroundingUtils { public static EventGrounding createEventGrounding() { EventGrounding eventGrounding = new EventGrounding(); + var messagingSettings = Utils + .getCoreConfigStorage() + .get() + .getMessagingSettings(); String topic = TOPIC_PREFIX + UUID.randomUUID().toString(); TopicDefinition topicDefinition = new SimpleTopicDefinition(topic); SpProtocol prioritizedProtocol = - BackendConfig.INSTANCE.getMessagingSettings().getPrioritizedProtocols().get(0); + messagingSettings.getPrioritizedProtocols().get(0); if (isPrioritized(prioritizedProtocol, JmsTransportProtocol.class)) { eventGrounding.setTransportProtocol( makeJmsTransportProtocol( - BackendConfig.INSTANCE.getJmsHost(), - BackendConfig.INSTANCE.getJmsPort(), + messagingSettings.getJmsHost(), + messagingSettings.getJmsPort(), topicDefinition)); } else if (isPrioritized(prioritizedProtocol, KafkaTransportProtocol.class)) { eventGrounding.setTransportProtocol( makeKafkaTransportProtocol( - BackendConfig.INSTANCE.getKafkaHost(), - BackendConfig.INSTANCE.getKafkaPort(), + messagingSettings.getKafkaHost(), + messagingSettings.getKafkaPort(), topicDefinition)); } else if (isPrioritized(prioritizedProtocol, MqttTransportProtocol.class)) { eventGrounding.setTransportProtocol( makeMqttTransportProtocol( - BackendConfig.INSTANCE.getMqttHost(), - BackendConfig.INSTANCE.getMqttPort(), + messagingSettings.getMqttHost(), + messagingSettings.getMqttPort(), topicDefinition)); } else if (isPrioritized(prioritizedProtocol, NatsTransportProtocol.class)) { eventGrounding.setTransportProtocol( makeNatsTransportProtocol( - BackendConfig.INSTANCE.getNatsHost(), - BackendConfig.INSTANCE.getNatsPort(), + messagingSettings.getNatsHost(), + messagingSettings.getNatsPort(), topicDefinition)); } else if (isPrioritized(prioritizedProtocol, PulsarTransportProtocol.class)) { eventGrounding.setTransportProtocol( makePulsarTransportProtocol( - BackendConfig.INSTANCE.getPulsarUrl(), + messagingSettings.getPulsarUrl(), topicDefinition ) ); diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/TransportFormatGenerator.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/TransportFormatGenerator.java index 5c9bf0b926..7038866af5 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/TransportFormatGenerator.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/TransportFormatGenerator.java @@ -17,8 +17,7 @@ */ package org.apache.streampipes.connect.management.util; -import org.apache.streampipes.config.backend.BackendConfig; -import org.apache.streampipes.model.config.SpDataFormat; +import org.apache.streampipes.model.configuration.SpDataFormat; import org.apache.streampipes.model.grounding.TransportFormat; import org.apache.streampipes.sdk.helpers.SupportedFormats; import org.apache.streampipes.vocabulary.MessageFormat; @@ -29,8 +28,9 @@ public class TransportFormatGenerator { public static TransportFormat getTransportFormat() { + var cfg = Utils.getCoreConfigStorage().get(); List supportedFormats = - BackendConfig.INSTANCE.getMessagingSettings().getPrioritizedFormats(); + cfg.getMessagingSettings().getPrioritizedFormats(); if (supportedFormats.size() > 0) { return new TransportFormat(supportedFormats.get(0).getMessageFormat()); diff --git a/ui/src/app/configuration/shared/streampipes-pe-container-configs.ts b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/Utils.java similarity index 66% rename from ui/src/app/configuration/shared/streampipes-pe-container-configs.ts rename to streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/Utils.java index bc3a211df8..030b839bf7 100644 --- a/ui/src/app/configuration/shared/streampipes-pe-container-configs.ts +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/Utils.java @@ -16,10 +16,17 @@ * */ -export class StreampipesPeContainerConifgs { - valueType: string; - isPassword: boolean; - description: string; - key: string; - value: string | number | boolean; +package org.apache.streampipes.connect.management.util; + +import org.apache.streampipes.storage.api.ISpCoreConfigurationStorage; +import org.apache.streampipes.storage.management.StorageDispatcher; + +public class Utils { + + public static ISpCoreConfigurationStorage getCoreConfigStorage() { + return StorageDispatcher + .INSTANCE + .getNoSqlStore() + .getSpCoreConfigurationStorage(); + } } diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/EventGroundingProcessor.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/EventGroundingProcessor.java index 4ad57762ac..83d508560a 100644 --- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/EventGroundingProcessor.java +++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/EventGroundingProcessor.java @@ -19,19 +19,30 @@ package org.apache.streampipes.export.utils; import org.apache.streampipes.commons.exceptions.SpRuntimeException; -import org.apache.streampipes.config.backend.BackendConfig; -import org.apache.streampipes.model.config.SpProtocol; +import org.apache.streampipes.model.configuration.MessagingSettings; +import org.apache.streampipes.model.configuration.SpProtocol; import org.apache.streampipes.model.grounding.KafkaTransportProtocol; import org.apache.streampipes.model.grounding.MqttTransportProtocol; import org.apache.streampipes.model.grounding.NatsTransportProtocol; import org.apache.streampipes.model.grounding.PulsarTransportProtocol; import org.apache.streampipes.model.grounding.TransportProtocol; +import org.apache.streampipes.storage.management.StorageDispatcher; public class EventGroundingProcessor { - SpProtocol configuredProtocol; + private final MessagingSettings messagingSettings; + private final SpProtocol configuredProtocol; + public EventGroundingProcessor() { - this.configuredProtocol = BackendConfig.INSTANCE.getMessagingSettings().getPrioritizedProtocols().get(0); + this.messagingSettings = StorageDispatcher + .INSTANCE + .getNoSqlStore() + .getSpCoreConfigurationStorage() + .get() + .getMessagingSettings(); + + this.configuredProtocol = messagingSettings + .getPrioritizedProtocols().get(0); } public TransportProtocol applyOverride(TransportProtocol protocol) { @@ -66,28 +77,28 @@ private boolean isProtocol(SpProtocol configuredProtocol, private KafkaTransportProtocol makeKafkaProtocol() { var protocol = new KafkaTransportProtocol(); - protocol.setBrokerHostname(BackendConfig.INSTANCE.getKafkaHost()); - protocol.setKafkaPort(BackendConfig.INSTANCE.getKafkaPort()); + protocol.setBrokerHostname(messagingSettings.getKafkaHost()); + protocol.setKafkaPort(messagingSettings.getKafkaPort()); return protocol; } private MqttTransportProtocol makeMqttProtocol() { var protocol = new MqttTransportProtocol(); - protocol.setBrokerHostname(BackendConfig.INSTANCE.getMqttHost()); - protocol.setPort(BackendConfig.INSTANCE.getMqttPort()); + protocol.setBrokerHostname(messagingSettings.getMqttHost()); + protocol.setPort(messagingSettings.getMqttPort()); return protocol; } private NatsTransportProtocol makeNatsProtocol() { var protocol = new NatsTransportProtocol(); - protocol.setBrokerHostname(BackendConfig.INSTANCE.getNatsHost()); - protocol.setPort(BackendConfig.INSTANCE.getNatsPort()); + protocol.setBrokerHostname(messagingSettings.getNatsHost()); + protocol.setPort(messagingSettings.getNatsPort()); return protocol; } private PulsarTransportProtocol makePulsarProtocol() { var protocol = new PulsarTransportProtocol(); - protocol.setBrokerHostname(BackendConfig.INSTANCE.getNatsHost()); + protocol.setBrokerHostname(messagingSettings.getNatsHost()); return protocol; } diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/config/IConfigExtractor.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/config/IConfigExtractor.java index f423d5eb7d..e1a6727486 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/config/IConfigExtractor.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/config/IConfigExtractor.java @@ -18,10 +18,21 @@ package org.apache.streampipes.extensions.api.config; -import org.apache.streampipes.svcdiscovery.api.SpConfig; +import org.apache.streampipes.model.extensions.configuration.ConfigItem; import java.io.Serializable; public interface IConfigExtractor extends Serializable { - SpConfig getConfig(); + + boolean getBoolean(String key); + + int getInteger(String key); + + double getDouble(String key); + + String getString(String key); + + T getObject(String key, Class clazz, T defaultValue); + + ConfigItem getConfigItem(String key); } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/client/StreamPipesClientRuntimeConnectionResolver.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/client/StreamPipesClientRuntimeConnectionResolver.java index cd1735f18a..6e1ea3811d 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/client/StreamPipesClientRuntimeConnectionResolver.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/client/StreamPipesClientRuntimeConnectionResolver.java @@ -24,15 +24,11 @@ import org.apache.streampipes.commons.environment.Environments; import org.apache.streampipes.commons.exceptions.SpRuntimeException; import org.apache.streampipes.commons.networking.Networking; -import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceGroups; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTags; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.UnknownHostException; -import java.util.Collections; import java.util.List; public class StreamPipesClientRuntimeConnectionResolver implements ClientConnectionUrlResolver { @@ -55,7 +51,7 @@ public String getBaseUrl() throws SpRuntimeException { if (baseUrls.size() > 0) { if (env.getSpDebug().getValueOrDefault()) { try { - return "http://" + Networking.getHostname() + ":" + 8030; + return buildBaseUrl(Networking.getHostname()); } catch (UnknownHostException e) { LOG.error("Could not auto-resolve host address - using http://localhost:8030"); return "http://localhost:8030"; @@ -76,12 +72,14 @@ private String getClientApiSecret() { } private List findClientServices() { - return SpServiceDiscovery - .getServiceDiscovery() - .getServiceEndpoints( - DefaultSpServiceGroups.CORE, - true, - Collections.singletonList(DefaultSpServiceTags.STREAMPIPES_CLIENT.asString()) - ); + return List.of(buildBaseUrl(env.getSpCoreHost().getValueOrDefault())); + } + + private String buildBaseUrl(String host) { + return env.getSpCoreScheme().getValueOrDefault() + + "://" + + host + + ":" + + env.getSpCorePort().getValueOrDefault(); } } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/config/ConfigExtractor.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/config/ConfigExtractor.java index 4b173b592e..dd1fef5f6d 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/config/ConfigExtractor.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/config/ConfigExtractor.java @@ -18,23 +18,63 @@ package org.apache.streampipes.extensions.management.config; import org.apache.streampipes.extensions.api.config.IConfigExtractor; -import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.SpConfig; +import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver; +import org.apache.streampipes.model.extensions.configuration.ConfigItem; +import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; public class ConfigExtractor implements IConfigExtractor { - private SpConfig config; + private final SpServiceConfiguration serviceConfig; - private ConfigExtractor(String serviceGroup) { - this.config = SpServiceDiscovery.getSpConfig(serviceGroup); + private ConfigExtractor(SpServiceConfiguration serviceConfig) { + this.serviceConfig = serviceConfig; } - public static ConfigExtractor from(String serviceId) { - return new ConfigExtractor(serviceId); + public static ConfigExtractor from(String serviceGroup) { + var client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); + var serviceConfig = client.adminApi().getServiceConfiguration(serviceGroup); + return new ConfigExtractor(serviceConfig); + } + + public static ConfigExtractor from(SpServiceConfiguration serviceConfig) { + return new ConfigExtractor(serviceConfig); + } + + @Override + public boolean getBoolean(String key) { + return Boolean.parseBoolean(getString(key)); } @Override - public SpConfig getConfig() { - return this.config; + public int getInteger(String key) { + return Integer.parseInt(getString(key)); + } + + @Override + public double getDouble(String key) { + return Double.parseDouble(getString(key)); + } + + @Override + public String getString(String key) { + return getItem(key).getValue(); + } + + @Override + public T getObject(String key, Class clazz, T defaultValue) { + return null; + } + + @Override + public ConfigItem getConfigItem(String key) { + return getItem(key); + } + + private ConfigItem getItem(String key) { + return serviceConfig.getConfigs() + .stream() + .filter(c -> c.getKey().equals(key)) + .findFirst() + .orElseThrow(IllegalArgumentException::new); } } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java index 4042b4074e..5f0ae11026 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/AdapterPipelineGenerator.java @@ -18,10 +18,10 @@ package org.apache.streampipes.extensions.management.connect.adapter; -import org.apache.streampipes.config.backend.BackendConfig; import org.apache.streampipes.connect.shared.AdapterPipelineGeneratorBase; import org.apache.streampipes.connect.shared.preprocessing.elements.TransformStreamAdapterElement; import org.apache.streampipes.connect.shared.preprocessing.transform.stream.DuplicateFilterPipelineElement; +import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver; import org.apache.streampipes.extensions.management.connect.adapter.model.pipeline.AdapterPipeline; import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.SendToBrokerAdapterSink; import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.SendToJmsAdapterSink; @@ -29,7 +29,8 @@ import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.SendToMqttAdapterSink; import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.SendToNatsAdapterSink; import org.apache.streampipes.extensions.management.connect.adapter.preprocessing.elements.SendToPulsarAdapterSink; -import org.apache.streampipes.model.config.SpProtocol; +import org.apache.streampipes.model.configuration.MessagingSettings; +import org.apache.streampipes.model.configuration.SpProtocol; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.grounding.JmsTransportProtocol; import org.apache.streampipes.model.grounding.KafkaTransportProtocol; @@ -70,7 +71,7 @@ public AdapterPipeline generatePipeline(AdapterDescription adapterDescription) { private SendToBrokerAdapterSink getAdapterSink(AdapterDescription adapterDescription) { var prioritizedProtocol = - BackendConfig.INSTANCE.getMessagingSettings().getPrioritizedProtocols().get(0); + getMessagingSettings().getPrioritizedProtocols().get(0); if (isPrioritized(prioritizedProtocol, JmsTransportProtocol.class)) { return new SendToJmsAdapterSink(adapterDescription); @@ -89,4 +90,9 @@ private boolean isPrioritized(SpProtocol prioritizedProtocol, Class protocolClass) { return prioritizedProtocol.getProtocolClass().equals(protocolClass.getCanonicalName()); } + + private MessagingSettings getMessagingSettings() { + var client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); + return client.adminApi().getMessagingSettings(); + } } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/context/AdapterContextGenerator.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/context/AdapterContextGenerator.java index 77e24c234a..f774a41401 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/context/AdapterContextGenerator.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/context/AdapterContextGenerator.java @@ -37,6 +37,4 @@ public IAdapterRuntimeContext makeRuntimeContext(String adapterInstanceId) { public IAdapterGuessSchemaContext makeGuessSchemaContext() { return new SpAdapterGuessSchemaContext(makeConfigExtractor(), makeStreamPipesClient()); } - - } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/context/SpAdapterGuessSchemaContext.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/context/SpAdapterGuessSchemaContext.java index b746accb13..f80ae4ceeb 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/context/SpAdapterGuessSchemaContext.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/context/SpAdapterGuessSchemaContext.java @@ -19,22 +19,22 @@ package org.apache.streampipes.extensions.management.context; import org.apache.streampipes.client.StreamPipesClient; +import org.apache.streampipes.extensions.api.config.IConfigExtractor; import org.apache.streampipes.extensions.api.connect.context.IAdapterGuessSchemaContext; -import org.apache.streampipes.extensions.management.config.ConfigExtractor; public class SpAdapterGuessSchemaContext implements IAdapterGuessSchemaContext { - protected ConfigExtractor configExtractor; + protected IConfigExtractor configExtractor; protected StreamPipesClient streamPipesClient; - public SpAdapterGuessSchemaContext(ConfigExtractor configExtractor, + public SpAdapterGuessSchemaContext(IConfigExtractor configExtractor, StreamPipesClient streamPipesClient) { this.configExtractor = configExtractor; this.streamPipesClient = streamPipesClient; } @Override - public ConfigExtractor getConfigStore() { + public IConfigExtractor getConfigStore() { return configExtractor; } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/context/SpAdapterRuntimeContext.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/context/SpAdapterRuntimeContext.java index 2fc6fd95e6..2dea835420 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/context/SpAdapterRuntimeContext.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/context/SpAdapterRuntimeContext.java @@ -19,9 +19,9 @@ package org.apache.streampipes.extensions.management.context; import org.apache.streampipes.client.StreamPipesClient; +import org.apache.streampipes.extensions.api.config.IConfigExtractor; import org.apache.streampipes.extensions.api.connect.context.IAdapterRuntimeContext; import org.apache.streampipes.extensions.api.monitoring.IExtensionsLogger; -import org.apache.streampipes.extensions.management.config.ConfigExtractor; import java.io.Serializable; @@ -31,7 +31,7 @@ public class SpAdapterRuntimeContext extends SpAdapterGuessSchemaContext private final IExtensionsLogger extensionsLogger; public SpAdapterRuntimeContext(IExtensionsLogger extensionsLogger, - ConfigExtractor configExtractor, + IConfigExtractor configExtractor, StreamPipesClient streamPipesClient) { super(configExtractor, streamPipesClient); this.extensionsLogger = extensionsLogger; diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java index 80f4eca7a6..fed5a0baa4 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java @@ -33,9 +33,6 @@ import org.apache.streampipes.model.grounding.TransportFormat; import org.apache.streampipes.model.grounding.TransportProtocol; import org.apache.streampipes.model.util.Cloner; -import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.SpConfig; -import org.apache.streampipes.svcdiscovery.api.model.ConfigItem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,6 +68,7 @@ public class DeclarersSingleton implements IDeclarersSingleton { private List runtimeProviders; private String serviceId; + private String serviceGroup; private int port; private String route; @@ -99,11 +97,11 @@ public static DeclarersSingleton getInstance() { public void populate(String host, Integer port, SpServiceDefinition serviceDef) { this.serviceDefinition = serviceDef; - this.registerConfigs(serviceDef.getServiceGroup(), serviceDef.getServiceName(), serviceDef.getKvConfigs()); this.setHostName(host); this.setPort(port); this.addDeclarers(serviceDef.getDeclarers()); this.serviceId = serviceDef.getServiceId(); + this.serviceGroup = serviceDef.getServiceGroup(); this.registerProtocols(serviceDef.getProtocolDefinitionFactories()); this.registerDataFormats(serviceDef.getDataFormatFactories()); this.runtimeProviders = serviceDef.getRuntimeProviders(); @@ -111,15 +109,6 @@ public void populate(String host, Integer port, SpServiceDefinition serviceDef) serviceDef.getFunctions().forEach(f -> this.functions.put(f.getFunctionConfig().getFunctionId().getId(), f)); } - private void registerConfigs(String serviceGroup, - String serviceName, - Map configs) { - LOG.info("Registering {} configs in key/value store", configs.size()); - SpConfig spConfig = SpServiceDiscovery.getSpConfig(serviceGroup); - configs.values().forEach(spConfig::register); - spConfig.register(ConfigItem.from("SP_SERVICE_NAME", serviceName, "")); - } - public void addDeclarers(List> allPipelineElements) { allPipelineElements.forEach(this::add); } @@ -280,6 +269,10 @@ public SpServiceDefinition toServiceDefinition(String serviceId) { return serviceDef; } + public String getServiceGroup() { + return serviceGroup; + } + public SpServiceDefinition getServiceDefinition() { return this.serviceDefinition; } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java index 463ddaf375..ca22193907 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java @@ -23,12 +23,10 @@ import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement; import org.apache.streampipes.extensions.api.pe.runtime.IStreamPipesRuntimeProvider; import org.apache.streampipes.messaging.SpProtocolDefinitionFactory; -import org.apache.streampipes.svcdiscovery.api.model.ConfigItem; +import org.apache.streampipes.model.extensions.configuration.ConfigItem; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.UUID; public class SpServiceDefinition { @@ -46,7 +44,7 @@ public class SpServiceDefinition { private List adapters; - private Map kvConfigs; + private List kvConfigs; private List runtimeProviders; @@ -55,7 +53,7 @@ public SpServiceDefinition() { this.pipelineElements = new ArrayList<>(); this.dataFormatFactories = new ArrayList<>(); this.protocolDefinitionFactories = new ArrayList<>(); - this.kvConfigs = new HashMap<>(); + this.kvConfigs = new ArrayList<>(); this.functions = new ArrayList<>(); this.adapters = new ArrayList<>(); this.runtimeProviders = new ArrayList<>(); @@ -126,7 +124,7 @@ public void addDataFormatFactories(List factories) { } public void addConfig(ConfigItem configItem) { - this.kvConfigs.put(configItem.getKey(), configItem); + this.kvConfigs.add(configItem); } public void addAdapter(StreamPipesAdapter adapter) { @@ -161,7 +159,7 @@ public void setProtocolDefinitionFactories(List> this.protocolDefinitionFactories = protocolDefinitionFactories; } - public Map getKvConfigs() { + public List getKvConfigs() { return kvConfigs; } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java index 76f640393b..a350950c9e 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java @@ -23,7 +23,7 @@ import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement; import org.apache.streampipes.extensions.api.pe.runtime.IStreamPipesRuntimeProvider; import org.apache.streampipes.messaging.SpProtocolDefinitionFactory; -import org.apache.streampipes.svcdiscovery.api.model.ConfigItem; +import org.apache.streampipes.model.extensions.configuration.ConfigItem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,8 +126,8 @@ public SpServiceDefinitionBuilder registerMessagingProtocols(SpProtocolDefinitio public SpServiceDefinitionBuilder merge(SpServiceDefinition other) { this.serviceDefinition.addDeclarers(other.getDeclarers()); this.serviceDefinition.addAdapters(other.getAdapters()); - other.getKvConfigs().values().forEach(value -> { - if (this.serviceDefinition.getKvConfigs().containsKey(value.getKey())) { + other.getKvConfigs().forEach(value -> { + if (this.serviceDefinition.getKvConfigs().stream().anyMatch(c -> c.getKey().equals(value.getKey()))) { LOG.warn("Config key {} already exists and will be overridden by merge, which might lead to strange results.", value.getKey()); } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/RuntimeContextUtils.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/RuntimeContextUtils.java index 1823557b75..c8223431a7 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/RuntimeContextUtils.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/RuntimeContextUtils.java @@ -19,14 +19,15 @@ package org.apache.streampipes.extensions.management.util; import org.apache.streampipes.client.StreamPipesClient; +import org.apache.streampipes.extensions.api.config.IConfigExtractor; import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver; import org.apache.streampipes.extensions.management.config.ConfigExtractor; import org.apache.streampipes.extensions.management.init.DeclarersSingleton; public class RuntimeContextUtils { - public static ConfigExtractor makeConfigExtractor() { - var serviceId = DeclarersSingleton.getInstance().getServiceId(); + public static IConfigExtractor makeConfigExtractor() { + var serviceId = DeclarersSingleton.getInstance().getServiceGroup(); return ConfigExtractor.from(serviceId); } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/ServiceDefinitionUtil.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/ServiceDefinitionUtil.java index 104e7a8b78..e89858528c 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/ServiceDefinitionUtil.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/ServiceDefinitionUtil.java @@ -21,10 +21,10 @@ import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement; import org.apache.streampipes.model.SpDataStream; import org.apache.streampipes.model.base.NamedStreamPipesEntity; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.model.graph.DataProcessorDescription; import org.apache.streampipes.model.graph.DataSinkDescription; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceTag; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceTagPrefix; import java.util.Collection; import java.util.List; diff --git a/streampipes-extensions/streampipes-pipeline-elements-all-jvm/src/main/java/org/apache/streampipes/pe/jvm/config/AllPipelineElementsConfig.java b/streampipes-extensions/streampipes-pipeline-elements-all-jvm/src/main/java/org/apache/streampipes/pe/jvm/config/AllPipelineElementsConfig.java deleted file mode 100644 index ff7b411609..0000000000 --- a/streampipes-extensions/streampipes-pipeline-elements-all-jvm/src/main/java/org/apache/streampipes/pe/jvm/config/AllPipelineElementsConfig.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.pe.jvm.config; - -import org.apache.streampipes.extensions.api.PeConfig; -import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.SpConfig; - -public enum AllPipelineElementsConfig implements PeConfig { - INSTANCE; - - private SpConfig config; - - public static String serverUrl; - - private static final String service_id = "pe/org.apache.streampipes.processors.all.jvm"; - private static final String service_name = "Processors JVM (Bundle)"; - private static final String service_container_name = "pipeline-elements-all-jvm"; - - AllPipelineElementsConfig() { - config = SpServiceDiscovery.getSpConfig(service_id); - config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe esper"); - config.register(ConfigKeys.PORT, 8090, "Port for the pe esper"); - - config.register(ConfigKeys.SERVICE_NAME_KEY, service_name, "The name of the service"); - - } - - static { - serverUrl = - AllPipelineElementsConfig.INSTANCE.getHost() + ":" + AllPipelineElementsConfig.INSTANCE.getPort(); - } - - @Override - public String getHost() { - return config.getString(ConfigKeys.HOST); - } - - @Override - public int getPort() { - return config.getInteger(ConfigKeys.PORT); - } - - @Override - public String getId() { - return service_id; - } - - @Override - public String getName() { - return config.getString(ConfigKeys.SERVICE_NAME_KEY); - } - -} diff --git a/streampipes-extensions/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/latlong/processor/geocoder/googlemaps/GoogleMapsGeocoderProcessor.java b/streampipes-extensions/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/latlong/processor/geocoder/googlemaps/GoogleMapsGeocoderProcessor.java index e07af4cf60..65cb65c966 100644 --- a/streampipes-extensions/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/latlong/processor/geocoder/googlemaps/GoogleMapsGeocoderProcessor.java +++ b/streampipes-extensions/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/latlong/processor/geocoder/googlemaps/GoogleMapsGeocoderProcessor.java @@ -79,7 +79,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputC this.geocoderRequest = parameters.extractor().mappingPropertyValue(GEOCODER_REQUEST_KEY); - String googleMapsApiKey = runtimeContext.getConfigStore().getConfig().getString(ConfigKeys.GOOGLE_API_KEY); + String googleMapsApiKey = runtimeContext.getConfigStore().getString(ConfigKeys.GOOGLE_API_KEY); if (googleMapsApiKey == null || googleMapsApiKey.equals("")) { throw new SpRuntimeException("Could not start Geocoder. Did you forget to add a Google Maps API key?"); diff --git a/streampipes-extensions/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/latlong/processor/geocoder/googlemapsstatic/GoogleMapsStaticGeocoderProcessor.java b/streampipes-extensions/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/latlong/processor/geocoder/googlemapsstatic/GoogleMapsStaticGeocoderProcessor.java index 8f9f8cdcba..edf1414279 100644 --- a/streampipes-extensions/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/latlong/processor/geocoder/googlemapsstatic/GoogleMapsStaticGeocoderProcessor.java +++ b/streampipes-extensions/streampipes-processors-geo-jvm/src/main/java/org/apache/streampipes/processors/geo/jvm/latlong/processor/geocoder/googlemapsstatic/GoogleMapsStaticGeocoderProcessor.java @@ -80,7 +80,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputC this.staticGeocoderRequest = parameters.extractor().singleValueParameter(STATIC_GEOCODER_REQUEST_KEY, String.class); - String googleMapsApiKey = runtimeContext.getConfigStore().getConfig().getString(ConfigKeys.GOOGLE_API_KEY); + String googleMapsApiKey = runtimeContext.getConfigStore().getString(ConfigKeys.GOOGLE_API_KEY); if (googleMapsApiKey == null || googleMapsApiKey.equals("")) { throw new SpRuntimeException("Could not start Geocoder. Did you forget to add a Google Maps" + " API key?"); diff --git a/streampipes-extensions/streampipes-sources-vehicle-simulator/src/main/java/org/apache/streampipes/sources/vehicle/simulator/simulator/VehicleDataSimulator.java b/streampipes-extensions/streampipes-sources-vehicle-simulator/src/main/java/org/apache/streampipes/sources/vehicle/simulator/simulator/VehicleDataSimulator.java index 63a036b6a3..62fbad54f7 100644 --- a/streampipes-extensions/streampipes-sources-vehicle-simulator/src/main/java/org/apache/streampipes/sources/vehicle/simulator/simulator/VehicleDataSimulator.java +++ b/streampipes-extensions/streampipes-sources-vehicle-simulator/src/main/java/org/apache/streampipes/sources/vehicle/simulator/simulator/VehicleDataSimulator.java @@ -47,7 +47,7 @@ public VehicleDataSimulator() { private void initSimulation() { try { ConfigExtractor configExtractor = - ConfigExtractor.from(DeclarersSingleton.getInstance().getServiceDefinition().getServiceGroup()); + ConfigExtractor.from(DeclarersSingleton.getInstance().getServiceGroup()); SimulationConfig config = buildSimulationConfig(); Map workflows = buildSimWorkflows(config); String kafkaHost = getKafkaHost(configExtractor); @@ -78,12 +78,12 @@ private Map buildSimWorkflows(SimulationConfig confi private String getKafkaHost(ConfigExtractor configExtractor) { return env.getSpDebug().getValueOrDefault() - ? "localhost" : configExtractor.getConfig().getString(ConfigKeys.KAFKA_HOST); + ? "localhost" : configExtractor.getString(ConfigKeys.KAFKA_HOST); } private Integer getKafkaPort(ConfigExtractor configExtractor) { return env.getSpDebug().getValueOrDefault() - ? 9094 : configExtractor.getConfig().getInteger(ConfigKeys.KAFKA_PORT); + ? 9094 : configExtractor.getInteger(ConfigKeys.KAFKA_PORT); } @Override diff --git a/streampipes-extensions/streampipes-sources-vehicle-simulator/src/main/java/org/apache/streampipes/sources/vehicle/simulator/vehicle/streams/VehicleStream.java b/streampipes-extensions/streampipes-sources-vehicle-simulator/src/main/java/org/apache/streampipes/sources/vehicle/simulator/vehicle/streams/VehicleStream.java index d29c2bf371..44e0504397 100644 --- a/streampipes-extensions/streampipes-sources-vehicle-simulator/src/main/java/org/apache/streampipes/sources/vehicle/simulator/vehicle/streams/VehicleStream.java +++ b/streampipes-extensions/streampipes-sources-vehicle-simulator/src/main/java/org/apache/streampipes/sources/vehicle/simulator/vehicle/streams/VehicleStream.java @@ -46,8 +46,8 @@ public SpDataStream declareModel() { + "value of the vehicle's position"), "longitude", Geo.LNG)) .format(Formats.jsonFormat()) .protocol(Protocols.kafka( - configExtractor().getConfig().getString(ConfigKeys.KAFKA_HOST), - configExtractor().getConfig().getInteger(ConfigKeys.KAFKA_PORT), + configExtractor().getString(ConfigKeys.KAFKA_HOST), + configExtractor().getInteger(ConfigKeys.KAFKA_PORT), "org.apache.streampipes.examples.sources.vehicle")) .build(); } diff --git a/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/utils/WatertankDataSimulator.java b/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/utils/WatertankDataSimulator.java index 35dd6f2e03..48d124c35f 100644 --- a/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/utils/WatertankDataSimulator.java +++ b/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/utils/WatertankDataSimulator.java @@ -47,7 +47,7 @@ public WatertankDataSimulator() { private void initSimulation() { try { ConfigExtractor configExtractor = - ConfigExtractor.from(DeclarersSingleton.getInstance().getServiceDefinition().getServiceGroup()); + ConfigExtractor.from(DeclarersSingleton.getInstance().getServiceGroup()); SimulationConfig config = buildSimulationConfig(); Map workflows = buildSimWorkflows(config); String kafkaHost = getKafkaHost(configExtractor); @@ -69,12 +69,12 @@ private SimulationConfig buildSimulationConfig() throws IOException { private String getKafkaHost(ConfigExtractor configExtractor) { return env.getSpDebug().getValueOrDefault() - ? "localhost" : configExtractor.getConfig().getString(ConfigKeys.KAFKA_HOST); + ? "localhost" : configExtractor.getString(ConfigKeys.KAFKA_HOST); } private Integer getKafkaPort(ConfigExtractor configExtractor) { return env.getSpDebug().getValueOrDefault() - ? 9094 : configExtractor.getConfig().getInteger(ConfigKeys.KAFKA_PORT); + ? 9094 : configExtractor.getInteger(ConfigKeys.KAFKA_PORT); } private Map buildSimWorkflows(SimulationConfig config) throws IOException { diff --git a/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/FlowRate1Stream.java b/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/FlowRate1Stream.java index af6b2f5a89..1005ba03db 100644 --- a/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/FlowRate1Stream.java +++ b/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/FlowRate1Stream.java @@ -68,8 +68,8 @@ public SpDataStream declareModel() { .build()) .format(Formats.jsonFormat()) .protocol(Protocols.kafka( - configExtractor().getConfig().getString(ConfigKeys.KAFKA_HOST), - configExtractor().getConfig().getInteger(ConfigKeys.KAFKA_PORT), + configExtractor().getString(ConfigKeys.KAFKA_HOST), + configExtractor().getInteger(ConfigKeys.KAFKA_PORT), "org.apache.streampipes.examples.flowrate-1") ) .build(); diff --git a/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/FlowRate2Stream.java b/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/FlowRate2Stream.java index 023fdc8e7c..cfd4638acc 100644 --- a/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/FlowRate2Stream.java +++ b/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/FlowRate2Stream.java @@ -85,8 +85,8 @@ public SpDataStream declareModel() { .build()) .format(Formats.jsonFormat()) .protocol(Protocols.kafka( - configExtractor().getConfig().getString(ConfigKeys.KAFKA_HOST), - configExtractor().getConfig().getInteger(ConfigKeys.KAFKA_PORT), + configExtractor().getString(ConfigKeys.KAFKA_HOST), + configExtractor().getInteger(ConfigKeys.KAFKA_PORT), "org.apache.streampipes.examples.flowrate2")) .build(); } diff --git a/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/PressureTankStream.java b/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/PressureTankStream.java index fa7bfe71de..cbe4dbd0fa 100644 --- a/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/PressureTankStream.java +++ b/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/PressureTankStream.java @@ -57,8 +57,8 @@ public SpDataStream declareModel() { .build()) .format(Formats.jsonFormat()) .protocol(Protocols.kafka( - configExtractor().getConfig().getString(ConfigKeys.KAFKA_HOST), - configExtractor().getConfig().getInteger(ConfigKeys.KAFKA_PORT), + configExtractor().getString(ConfigKeys.KAFKA_HOST), + configExtractor().getInteger(ConfigKeys.KAFKA_PORT), "org.apache.streampipes.examples.pressuretank")) .build(); } diff --git a/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/WaterLevel1Stream.java b/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/WaterLevel1Stream.java index c516afc177..5e54e94297 100644 --- a/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/WaterLevel1Stream.java +++ b/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/WaterLevel1Stream.java @@ -71,8 +71,8 @@ public SpDataStream declareModel() { .build()) .format(Formats.jsonFormat()) .protocol(Protocols.kafka( - configExtractor().getConfig().getString(ConfigKeys.KAFKA_HOST), - configExtractor().getConfig().getInteger(ConfigKeys.KAFKA_PORT), + configExtractor().getString(ConfigKeys.KAFKA_HOST), + configExtractor().getInteger(ConfigKeys.KAFKA_PORT), "org.apache.streampipes.examples.waterlevel")) .build(); } diff --git a/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/WaterLevel2Stream.java b/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/WaterLevel2Stream.java index eece73b8a5..685ec156df 100644 --- a/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/WaterLevel2Stream.java +++ b/streampipes-extensions/streampipes-sources-watertank-simulator/src/main/java/org/apache/streampipes/sources/watertank/simulator/watertank/streams/WaterLevel2Stream.java @@ -63,8 +63,8 @@ public SpDataStream declareModel() { .build()) .format(Formats.jsonFormat()) .protocol(Protocols.kafka( - configExtractor().getConfig().getString(ConfigKeys.KAFKA_HOST), - configExtractor().getConfig().getInteger(ConfigKeys.KAFKA_PORT), + configExtractor().getString(ConfigKeys.KAFKA_HOST), + configExtractor().getInteger(ConfigKeys.KAFKA_PORT), "org.apache.streampipes.examples.waterlevel2")) .build(); } diff --git a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/svcdiscovery/AbstractConsulTest.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/svcdiscovery/AbstractConsulTest.java index 76f3444f04..0e7107f232 100644 --- a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/svcdiscovery/AbstractConsulTest.java +++ b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/svcdiscovery/AbstractConsulTest.java @@ -22,7 +22,7 @@ import org.apache.streampipes.commons.environment.variable.BooleanEnvironmentVariable; import org.apache.streampipes.commons.environment.variable.IntEnvironmentVariable; import org.apache.streampipes.commons.environment.variable.StringEnvironmentVariable; -import org.apache.streampipes.svcdiscovery.api.model.ConfigItem; +import org.apache.streampipes.model.extensions.configuration.ConfigItem; import org.mockito.Mockito; import org.testcontainers.consul.ConsulContainer; diff --git a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/svcdiscovery/ConsulKvManagementTest.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/svcdiscovery/ConsulKvManagementTest.java deleted file mode 100644 index 5b14c0a636..0000000000 --- a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/svcdiscovery/ConsulKvManagementTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.integration.svcdiscovery; - -import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.ISpKvManagement; -import org.apache.streampipes.svcdiscovery.api.model.ConfigItem; - -import com.google.gson.Gson; -import org.junit.Before; -import org.junit.Test; - -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -public class ConsulKvManagementTest extends AbstractConsulTest { - - private ISpKvManagement kvManagement; - - @Before - public void before() { - var environment = mockEnvironment(); - this.kvManagement = SpServiceDiscovery - .getKeyValueStore(environment); - } - - @Test - public void testKv() { - var test1Key = "a/test"; - var test2Key = "a/test2"; - var configItem1 = makeConfigItem(test1Key, "abc"); - var configItem2 = makeConfigItem(test2Key, "def"); - kvManagement.updateConfig(test1Key, serialize(configItem1), false); - kvManagement.updateConfig(test2Key, serialize(configItem2), false); - - Map result = kvManagement.getKeyValue("a"); - - assertEquals(2, result.size()); - assertEquals(serialize(configItem1), result.get(test1Key)); - assertEquals(serialize(configItem2), result.get(test2Key)); - } - - private String serialize(ConfigItem item) { - return new Gson().toJson(item); - } -} diff --git a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/svcdiscovery/ConsulSpServiceDiscoveryTest.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/svcdiscovery/ConsulSpServiceDiscoveryTest.java deleted file mode 100644 index 0a4093c30e..0000000000 --- a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/svcdiscovery/ConsulSpServiceDiscoveryTest.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.integration.svcdiscovery; - -import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceGroups; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTags; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceRegistrationRequest; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceTag; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceTagPrefix; - -import org.junit.Before; -import org.junit.Test; - -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class ConsulSpServiceDiscoveryTest extends AbstractConsulTest { - - private ISpServiceDiscovery serviceDiscovery; - - @Before - public void before() { - var env = mockEnvironment(); - this.serviceDiscovery = SpServiceDiscovery - .getServiceDiscovery(env); - } - - @Test - public void testServiceRegistration() { - - serviceDiscovery.registerService(makeRequest(makeServiceTag(), "abc")); - - var endpoints = getEndpoints(false); - assertEquals(1, endpoints.size()); - - serviceDiscovery.deregisterService("abc"); - - endpoints = getEndpoints(false); - assertEquals(0, endpoints.size()); - } - - @Test - public void testGetExtensionsServiceGroups() { - serviceDiscovery.registerService(makeRequest(List.of(DefaultSpServiceTags.PE, groupTag()), "abc")); - serviceDiscovery.registerService(makeRequest( - List.of(DefaultSpServiceTags.CORE, groupTag()), "def")); - - var serviceGroups = serviceDiscovery.getExtensionsServiceGroups(); - - assertEquals(1, serviceGroups.size()); - assertTrue(serviceGroups.containsKey("my-group")); - - serviceDiscovery.deregisterService("abc"); - serviceDiscovery.deregisterService("def"); - } - - private List getEndpoints(boolean healthy) { - return serviceDiscovery.getServiceEndpoints( - DefaultSpServiceGroups.EXT, - healthy, - List.of(makeServiceTag().get(0).asString())); - } - - private SpServiceRegistrationRequest makeRequest(List serviceTags, - String serviceId) { - var req = new SpServiceRegistrationRequest( - DefaultSpServiceGroups.EXT, - serviceId, - "localhost", - 80, - serviceTags, - "/" - ); - - return req; - } - - private SpServiceTag groupTag() { - return SpServiceTag.create(SpServiceTagPrefix.SP_GROUP, "my-group"); - } - - private List makeServiceTag() { - return List.of(DefaultSpServiceTags.PE); - } -} diff --git a/streampipes-mail/src/main/java/org/apache/streampipes/mail/AbstractMailer.java b/streampipes-mail/src/main/java/org/apache/streampipes/mail/AbstractMailer.java index 59d113b41e..b974929c2b 100644 --- a/streampipes-mail/src/main/java/org/apache/streampipes/mail/AbstractMailer.java +++ b/streampipes-mail/src/main/java/org/apache/streampipes/mail/AbstractMailer.java @@ -17,9 +17,9 @@ */ package org.apache.streampipes.mail; -import org.apache.streampipes.config.backend.BackendConfig; -import org.apache.streampipes.config.backend.model.EmailConfig; import org.apache.streampipes.mail.config.MailConfigurationBuilder; +import org.apache.streampipes.mail.utils.MailUtils; +import org.apache.streampipes.model.configuration.EmailConfig; import org.apache.streampipes.user.management.encryption.SecretEncryptionManager; import org.simplejavamail.api.email.Email; @@ -44,7 +44,7 @@ protected Mailer getMailer(EmailConfig config) { } protected EmailConfig getEmailConfig() { - EmailConfig config = BackendConfig.INSTANCE.getEmailConfig(); + EmailConfig config = MailUtils.getSpCoreConfiguration().getEmailConfig(); return getDecryptedEmailConfig(config); } diff --git a/streampipes-mail/src/main/java/org/apache/streampipes/mail/MailTester.java b/streampipes-mail/src/main/java/org/apache/streampipes/mail/MailTester.java index d39749a56b..9a017e0297 100644 --- a/streampipes-mail/src/main/java/org/apache/streampipes/mail/MailTester.java +++ b/streampipes-mail/src/main/java/org/apache/streampipes/mail/MailTester.java @@ -17,9 +17,9 @@ */ package org.apache.streampipes.mail; -import org.apache.streampipes.config.backend.model.EmailConfig; import org.apache.streampipes.mail.template.TestMailTemplate; import org.apache.streampipes.mail.utils.MailUtils; +import org.apache.streampipes.model.configuration.EmailConfig; import org.simplejavamail.api.email.Email; diff --git a/streampipes-mail/src/main/java/org/apache/streampipes/mail/config/MailConfigurationBuilder.java b/streampipes-mail/src/main/java/org/apache/streampipes/mail/config/MailConfigurationBuilder.java index 5efcbdb241..cdddd5d1eb 100644 --- a/streampipes-mail/src/main/java/org/apache/streampipes/mail/config/MailConfigurationBuilder.java +++ b/streampipes-mail/src/main/java/org/apache/streampipes/mail/config/MailConfigurationBuilder.java @@ -17,7 +17,7 @@ */ package org.apache.streampipes.mail.config; -import org.apache.streampipes.config.backend.model.EmailConfig; +import org.apache.streampipes.model.configuration.EmailConfig; import org.simplejavamail.api.mailer.Mailer; import org.simplejavamail.api.mailer.config.TransportStrategy; @@ -60,10 +60,10 @@ public Mailer buildMailerFromConfig(EmailConfig config) { } private TransportStrategy toTransportStrategy( - org.apache.streampipes.config.backend.model.TransportStrategy strategy) { - if (strategy == org.apache.streampipes.config.backend.model.TransportStrategy.SMTP) { + org.apache.streampipes.model.configuration.TransportStrategy strategy) { + if (strategy == org.apache.streampipes.model.configuration.TransportStrategy.SMTP) { return TransportStrategy.SMTP; - } else if (strategy == org.apache.streampipes.config.backend.model.TransportStrategy.SMTPS) { + } else if (strategy == org.apache.streampipes.model.configuration.TransportStrategy.SMTPS) { return TransportStrategy.SMTPS; } else { return TransportStrategy.SMTP_TLS; diff --git a/streampipes-mail/src/main/java/org/apache/streampipes/mail/utils/MailUtils.java b/streampipes-mail/src/main/java/org/apache/streampipes/mail/utils/MailUtils.java index 317f534966..4a66194365 100644 --- a/streampipes-mail/src/main/java/org/apache/streampipes/mail/utils/MailUtils.java +++ b/streampipes-mail/src/main/java/org/apache/streampipes/mail/utils/MailUtils.java @@ -18,8 +18,9 @@ package org.apache.streampipes.mail.utils; import org.apache.streampipes.commons.resources.Resources; -import org.apache.streampipes.config.backend.BackendConfig; -import org.apache.streampipes.config.backend.model.GeneralConfig; +import org.apache.streampipes.model.configuration.GeneralConfig; +import org.apache.streampipes.model.configuration.SpCoreConfiguration; +import org.apache.streampipes.storage.management.StorageDispatcher; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -27,16 +28,20 @@ public class MailUtils { public static String extractBaseUrl() { - GeneralConfig config = BackendConfig.INSTANCE.getGeneralConfig(); + GeneralConfig config = getSpCoreConfiguration().getGeneralConfig(); return config.getProtocol() + "://" + config.getHostname() + ":" + config.getPort(); } public static String extractAppName() { - return BackendConfig.INSTANCE.getGeneralConfig().getAppName(); + return getSpCoreConfiguration().getGeneralConfig().getAppName(); } public static String readResourceFileToString(String filename) throws IOException { return Resources.asString(filename, StandardCharsets.UTF_8); } + + public static SpCoreConfiguration getSpCoreConfiguration() { + return StorageDispatcher.INSTANCE.getNoSqlStore().getSpCoreConfigurationStorage().get(); + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultMessagingSettings.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultMessagingSettings.java new file mode 100644 index 0000000000..d2f21ea0cb --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultMessagingSettings.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.configuration; + +import org.apache.streampipes.commons.environment.Environments; + +import java.util.Arrays; +import java.util.List; + +public class DefaultMessagingSettings { + + public MessagingSettings make() { + List protocolList; + var env = Environments.getEnvironment(); + if (env.getPrioritizedProtocol().exists()) { + protocolList = switch (env.getPrioritizedProtocol().getValueOrDefault().toLowerCase()) { + case "mqtt" -> + Arrays.asList(SpProtocol.MQTT, SpProtocol.KAFKA, SpProtocol.JMS, SpProtocol.NATS, SpProtocol.PULSAR); + case "kafka" -> + Arrays.asList(SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS, SpProtocol.PULSAR); + case "jms" -> + Arrays.asList(SpProtocol.JMS, SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.NATS, SpProtocol.PULSAR); + case "nats" -> + Arrays.asList(SpProtocol.NATS, SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.PULSAR); + case "pulsar" -> + Arrays.asList(SpProtocol.PULSAR, SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS); + default -> Arrays.asList(SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS, SpProtocol.PULSAR); + }; + } else { + protocolList = + Arrays.asList(SpProtocol.KAFKA, SpProtocol.MQTT, SpProtocol.JMS, SpProtocol.NATS, SpProtocol.PULSAR); + } + + var defaultSettings = new MessagingSettings( + 1638400, + 5000012, + 20, + 2, + Arrays.asList(SpDataFormat.JSON, SpDataFormat.CBOR, SpDataFormat.FST, SpDataFormat.SMILE), + protocolList); + + defaultSettings.setJmsHost("activemq"); + defaultSettings.setJmsPort(61616); + + defaultSettings.setMqttHost("mosquitto"); + defaultSettings.setMqttPort(1883); + + defaultSettings.setNatsHost("nats"); + defaultSettings.setNatsPort(4222); + + defaultSettings.setKafkaHost("kafka"); + defaultSettings.setKafkaPort(9092); + + defaultSettings.setPulsarUrl("pulsar://localhost:6650"); + + defaultSettings.setZookeeperHost("zookeeper"); + defaultSettings.setZookeeperPort(2181); + return defaultSettings; + } +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultSpCoreConfiguration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultSpCoreConfiguration.java new file mode 100644 index 0000000000..044d1f5c42 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultSpCoreConfiguration.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.configuration; + +import org.apache.streampipes.commons.environment.Environment; +import org.apache.streampipes.commons.environment.Environments; +import org.apache.streampipes.commons.random.TokenGenerator; + +import java.io.File; + +public class DefaultSpCoreConfiguration { + + public SpCoreConfiguration make() { + var coreCfg = new SpCoreConfiguration(); + + coreCfg.setGeneralConfig(new GeneralConfig()); + coreCfg.setMessagingSettings(new DefaultMessagingSettings().make()); + coreCfg.setEmailConfig(EmailConfig.fromDefaults()); + coreCfg.setFilesDir(makeFileLocation()); + coreCfg.setAssetDir(makeAssetLocation()); + coreCfg.setLocalAuthConfig(LocalAuthConfig.fromDefaults(getJwtSecret())); + + return coreCfg; + } + + private String makeAssetLocation() { + return makeStreamPipesHomeLocation() + + "assets"; + } + + private String makeFileLocation() { + return makeStreamPipesHomeLocation() + + "files"; + } + + private String makeStreamPipesHomeLocation() { + var userDefinedAssetDir = getEnvironment().getCoreAssetBaseDir(); + var assetDirAppendix = getSpAssetDirAppendix(); + if (userDefinedAssetDir.exists()) { + return userDefinedAssetDir.getValue() + assetDirAppendix; + } else { + return System.getProperty("user.home") + assetDirAppendix; + } + } + + private String getJwtSecret() { + var env = getEnvironment(); + return env.getJwtSecret().getValueOrResolve(this::makeDefaultJwtSecret); + } + + private String makeDefaultJwtSecret() { + return TokenGenerator.generateNewToken(); + } + + private Environment getEnvironment() { + return Environments.getEnvironment(); + } + + private String getSpAssetDirAppendix() { + return File.separator + ".streampipes" + File.separator; + } + +} diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/EmailConfig.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/EmailConfig.java similarity index 98% rename from streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/EmailConfig.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/configuration/EmailConfig.java index ca247c4c21..57e243fb13 100644 --- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/EmailConfig.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/EmailConfig.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.config.backend.model; +package org.apache.streampipes.model.configuration; public class EmailConfig { diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/GeneralConfig.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/GeneralConfig.java similarity index 98% rename from streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/GeneralConfig.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/configuration/GeneralConfig.java index e2d0dfca3f..2f3968d966 100644 --- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/GeneralConfig.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/GeneralConfig.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.config.backend.model; +package org.apache.streampipes.model.configuration; import java.util.List; diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/JwtSigningMode.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/JwtSigningMode.java similarity index 94% rename from streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/JwtSigningMode.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/configuration/JwtSigningMode.java index 7bc74e1833..d48fe6deee 100644 --- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/JwtSigningMode.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/JwtSigningMode.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.config.backend.model; +package org.apache.streampipes.model.configuration; public enum JwtSigningMode { HMAC, diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/LocalAuthConfig.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/LocalAuthConfig.java similarity index 98% rename from streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/LocalAuthConfig.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/configuration/LocalAuthConfig.java index 044eb0efcd..3137f0d7b9 100644 --- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/LocalAuthConfig.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/LocalAuthConfig.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.config.backend.model; +package org.apache.streampipes.model.configuration; public class LocalAuthConfig { diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/config/MessagingSettings.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/MessagingSettings.java similarity index 54% rename from streampipes-model/src/main/java/org/apache/streampipes/model/config/MessagingSettings.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/configuration/MessagingSettings.java index 277aa3bf32..edad5c1a50 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/config/MessagingSettings.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/MessagingSettings.java @@ -15,10 +15,14 @@ * limitations under the License. * */ -package org.apache.streampipes.model.config; +package org.apache.streampipes.model.configuration; +import org.apache.streampipes.model.shared.annotation.TsModel; + +import java.util.ArrayList; import java.util.List; +@TsModel public class MessagingSettings { private Integer batchSize; @@ -29,7 +33,24 @@ public class MessagingSettings { private List prioritizedFormats; private List prioritizedProtocols; + private String jmsHost; + private int jmsPort; + + private String mqttHost; + private int mqttPort; + + private String natsHost; + private int natsPort; + + private String kafkaHost; + private int kafkaPort; + private String pulsarUrl; + + private String zookeeperHost; + private int zookeeperPort; + + private List supportedProtocols; public MessagingSettings(Integer batchSize, Integer messageMaxBytes, @@ -43,6 +64,7 @@ public MessagingSettings(Integer batchSize, this.acks = acks; this.prioritizedFormats = prioritizedFormats; this.prioritizedProtocols = prioritizedProtocols; + this.supportedProtocols = new ArrayList<>(); } public MessagingSettings() { @@ -96,4 +118,100 @@ public List getPrioritizedProtocols() { public void setPrioritizedProtocols(List prioritizedProtocols) { this.prioritizedProtocols = prioritizedProtocols; } + + public String getJmsHost() { + return jmsHost; + } + + public void setJmsHost(String jmsHost) { + this.jmsHost = jmsHost; + } + + public int getJmsPort() { + return jmsPort; + } + + public void setJmsPort(int jmsPort) { + this.jmsPort = jmsPort; + } + + public String getMqttHost() { + return mqttHost; + } + + public void setMqttHost(String mqttHost) { + this.mqttHost = mqttHost; + } + + public int getMqttPort() { + return mqttPort; + } + + public void setMqttPort(int mqttPort) { + this.mqttPort = mqttPort; + } + + public String getNatsHost() { + return natsHost; + } + + public void setNatsHost(String natsHost) { + this.natsHost = natsHost; + } + + public int getNatsPort() { + return natsPort; + } + + public void setNatsPort(int natsPort) { + this.natsPort = natsPort; + } + + public String getKafkaHost() { + return kafkaHost; + } + + public void setKafkaHost(String kafkaHost) { + this.kafkaHost = kafkaHost; + } + + public int getKafkaPort() { + return kafkaPort; + } + + public void setKafkaPort(int kafkaPort) { + this.kafkaPort = kafkaPort; + } + + public String getPulsarUrl() { + return pulsarUrl; + } + + public void setPulsarUrl(String pulsarUrl) { + this.pulsarUrl = pulsarUrl; + } + + public String getZookeeperHost() { + return zookeeperHost; + } + + public void setZookeeperHost(String zookeeperHost) { + this.zookeeperHost = zookeeperHost; + } + + public int getZookeeperPort() { + return zookeeperPort; + } + + public void setZookeeperPort(int zookeeperPort) { + this.zookeeperPort = zookeeperPort; + } + + public List getSupportedProtocols() { + return supportedProtocols; + } + + public void setSupportedProtocols(List supportedProtocols) { + this.supportedProtocols = supportedProtocols; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java new file mode 100644 index 0000000000..32ea21d4ee --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.configuration; + +import com.google.gson.annotations.SerializedName; + +public class SpCoreConfiguration { + + public static final String ID = "core"; + + protected @SerializedName("_rev") String rev; + private @SerializedName("_id") String id = ID; + + private MessagingSettings messagingSettings; + private LocalAuthConfig localAuthConfig; + private EmailConfig emailConfig; + private GeneralConfig generalConfig; + + private boolean isConfigured; + + private String assetDir; + private String filesDir; + + public SpCoreConfiguration() { + } + + public String getRev() { + return rev; + } + + public void setRev(String rev) { + this.rev = rev; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public MessagingSettings getMessagingSettings() { + return messagingSettings; + } + + public void setMessagingSettings(MessagingSettings messagingSettings) { + this.messagingSettings = messagingSettings; + } + + public LocalAuthConfig getLocalAuthConfig() { + return localAuthConfig; + } + + public void setLocalAuthConfig(LocalAuthConfig localAuthConfig) { + this.localAuthConfig = localAuthConfig; + } + + public EmailConfig getEmailConfig() { + return emailConfig; + } + + public void setEmailConfig(EmailConfig emailConfig) { + this.emailConfig = emailConfig; + } + + public GeneralConfig getGeneralConfig() { + return generalConfig; + } + + public void setGeneralConfig(GeneralConfig generalConfig) { + this.generalConfig = generalConfig; + } + + public boolean isConfigured() { + return isConfigured; + } + + public void setConfigured(boolean configured) { + isConfigured = configured; + } + + public String getAssetDir() { + return assetDir; + } + + public void setAssetDir(String assetDir) { + this.assetDir = assetDir; + } + + public String getFilesDir() { + return filesDir; + } + + public void setFilesDir(String filesDir) { + this.filesDir = filesDir; + } +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/config/SpDataFormat.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpDataFormat.java similarity index 96% rename from streampipes-model/src/main/java/org/apache/streampipes/model/config/SpDataFormat.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpDataFormat.java index c409eeaab0..a2c5b6bf38 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/config/SpDataFormat.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpDataFormat.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.model.config; +package org.apache.streampipes.model.configuration; import org.apache.streampipes.vocabulary.MessageFormat; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/config/SpProtocol.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpProtocol.java similarity index 96% rename from streampipes-model/src/main/java/org/apache/streampipes/model/config/SpProtocol.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpProtocol.java index af7af72639..657ae968db 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/config/SpProtocol.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpProtocol.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.model.config; +package org.apache.streampipes.model.configuration; public enum SpProtocol { diff --git a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/TransportStrategy.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/TransportStrategy.java similarity index 94% rename from streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/TransportStrategy.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/configuration/TransportStrategy.java index ffb701eee6..4155f789db 100644 --- a/streampipes-config/src/main/java/org/apache/streampipes/config/backend/model/TransportStrategy.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/TransportStrategy.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.config.backend.model; +package org.apache.streampipes.model.configuration; public enum TransportStrategy { SMTP, diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/ConfigItem.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/ConfigItem.java similarity index 98% rename from streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/ConfigItem.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/ConfigItem.java index 803f4e43b9..e0bd898929 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/ConfigItem.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/ConfigItem.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.svcdiscovery.api.model; +package org.apache.streampipes.model.extensions.configuration; import java.util.Objects; diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/ConfigItemUtils.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/ConfigItemUtils.java similarity index 94% rename from streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/ConfigItemUtils.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/ConfigItemUtils.java index f1251b96b6..df73f6d1f9 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/ConfigItemUtils.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/ConfigItemUtils.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.svcdiscovery.api.model; +package org.apache.streampipes.model.extensions.configuration; public class ConfigItemUtils { diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/ConfigurationScope.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/ConfigurationScope.java similarity index 93% rename from streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/ConfigurationScope.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/ConfigurationScope.java index 094cb0c244..c0cc5c5485 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/ConfigurationScope.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/ConfigurationScope.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.svcdiscovery.api.model; +package org.apache.streampipes.model.extensions.configuration; public enum ConfigurationScope { CONTAINER_STARTUP_CONFIG, diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/SpServiceConfiguration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/SpServiceConfiguration.java new file mode 100644 index 0000000000..9cc4319d9a --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/SpServiceConfiguration.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.extensions.configuration; + +import org.apache.streampipes.model.shared.annotation.TsModel; + +import com.google.gson.annotations.SerializedName; + +import java.util.List; + +@TsModel +public class SpServiceConfiguration { + + protected @SerializedName("_rev") String rev; + private @SerializedName("_id") String serviceGroup; + + String serviceName; + List configs; + + public SpServiceConfiguration() { + } + + public SpServiceConfiguration(String serviceGroup, + String serviceName, + List configs) { + this.serviceGroup = serviceGroup; + this.serviceName = serviceName; + this.configs = configs; + } + + public String getServiceGroup() { + return serviceGroup; + } + + public void setServiceGroup(String serviceGroup) { + this.serviceGroup = serviceGroup; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public List getConfigs() { + return configs; + } + + public void setConfigs(List configs) { + this.configs = configs; + } + + public String getRev() { + return rev; + } + + public void setRev(String rev) { + this.rev = rev; + } +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceRegistration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceRegistration.java new file mode 100644 index 0000000000..2f731ba742 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceRegistration.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.model.extensions.svcdiscovery; + +import org.apache.streampipes.model.shared.annotation.TsModel; + +import com.google.gson.annotations.SerializedName; + +import java.util.List; + +@TsModel +public class SpServiceRegistration { + + private String svcType; + private String svcGroup; + + protected @SerializedName("_rev") String rev; + private @SerializedName("_id") String svcId; + private String scheme = "http"; + private String host; + private int port; + private List tags; + private String healthCheckPath; + private boolean healthy = true; + private long firstTimeSeenUnhealthy = 0; + + public SpServiceRegistration() { + } + + public SpServiceRegistration(String svcType, + String svcGroup, + String svcId, + String host, + int port, + List tags, + String healthCheckPath) { + this.svcType = svcType; + this.svcGroup = svcGroup; + this.svcId = svcId; + this.host = host; + this.port = port; + this.tags = tags; + this.healthCheckPath = healthCheckPath; + } + + public static SpServiceRegistration from(String svcType, + String svcGroup, + String svcId, + String host, + Integer port, + List tags) { + return new SpServiceRegistration(svcType, svcGroup, svcId, host, port, tags, ""); + } + + public static SpServiceRegistration from(String svcType, + String svcGroup, + String svcId, + String host, + Integer port, + List tags, + String healthCheckPath) { + return new SpServiceRegistration(svcType, svcGroup, svcId, host, port, tags, healthCheckPath); + } + + public String getSvcGroup() { + return svcGroup; + } + + public void setSvcGroup(String svcGroup) { + this.svcGroup = svcGroup; + } + + public String getSvcId() { + return svcId; + } + + public void setSvcId(String svcId) { + this.svcId = svcId; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public List getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public String getHealthCheckPath() { + return healthCheckPath; + } + + public void setHealthCheckPath(String healthCheckPath) { + this.healthCheckPath = healthCheckPath; + } + + public String getRev() { + return rev; + } + + public void setRev(String rev) { + this.rev = rev; + } + + public boolean isHealthy() { + return healthy; + } + + public void setHealthy(boolean healthy) { + this.healthy = healthy; + } + + public String getScheme() { + return scheme; + } + + public void setScheme(String scheme) { + this.scheme = scheme; + } + + public long getFirstTimeSeenUnhealthy() { + return firstTimeSeenUnhealthy; + } + + public void setFirstTimeSeenUnhealthy(long firstTimeSeenUnhealthy) { + this.firstTimeSeenUnhealthy = firstTimeSeenUnhealthy; + } + + public String getSvcType() { + return svcType; + } + + public void setSvcType(String svcType) { + this.svcType = svcType; + } +} diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceTag.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTag.java similarity index 74% rename from streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceTag.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTag.java index baabf90f33..15feea7113 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceTag.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTag.java @@ -15,13 +15,17 @@ * limitations under the License. * */ -package org.apache.streampipes.svcdiscovery.api.model; +package org.apache.streampipes.model.extensions.svcdiscovery; public class SpServiceTag { private static final String COLON = ":"; - private final SpServiceTagPrefix prefix; - private final String value; + private SpServiceTagPrefix prefix; + private String value; + + public SpServiceTag() { + + } private SpServiceTag(SpServiceTagPrefix prefix, String value) { this.prefix = prefix; @@ -36,4 +40,20 @@ public static SpServiceTag create(SpServiceTagPrefix prefix, public String asString() { return prefix.asString() + COLON + value; } + + public SpServiceTagPrefix getPrefix() { + return prefix; + } + + public void setPrefix(SpServiceTagPrefix prefix) { + this.prefix = prefix; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } } diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceTagPrefix.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTagPrefix.java similarity index 94% rename from streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceTagPrefix.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTagPrefix.java index ecdbbaea4f..006a00af29 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceTagPrefix.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTagPrefix.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.svcdiscovery.api.model; +package org.apache.streampipes.model.extensions.svcdiscovery; public enum SpServiceTagPrefix { SYSTEM("sys"), diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetConstants.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetConstants.java index eda79448f0..67ad1e1986 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetConstants.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetConstants.java @@ -17,10 +17,15 @@ */ package org.apache.streampipes.manager.assets; -import org.apache.streampipes.config.backend.BackendConfig; +import org.apache.streampipes.storage.management.StorageDispatcher; public class AssetConstants { - public static final String ASSET_BASE_DIR = BackendConfig.INSTANCE.getAssetDir(); + public static final String ASSET_BASE_DIR = StorageDispatcher + .INSTANCE + .getNoSqlStore() + .getSpCoreConfigurationStorage() + .get() + .getAssetDir(); } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java index 8b47fae45c..970d81a0d3 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java @@ -20,7 +20,7 @@ import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; import org.apache.streampipes.model.base.NamedStreamPipesEntity; import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceGroups; +import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTypes; import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; import org.slf4j.Logger; @@ -59,7 +59,7 @@ private List getServiceEndpoints() { return SpServiceDiscovery .getServiceDiscovery() .getServiceEndpoints( - DefaultSpServiceGroups.EXT, + DefaultSpServiceTypes.EXT, true, Collections .singletonList( diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointProvider.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointProvider.java index 92578a178d..6e76fedfe7 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointProvider.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointProvider.java @@ -18,15 +18,8 @@ package org.apache.streampipes.manager.execution.endpoint; -import org.apache.streampipes.commons.constants.GlobalStreamPipesConstants; import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; import org.apache.streampipes.model.base.InvocableStreamPipesEntity; -import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceGroups; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTags; - -import java.util.Collections; -import java.util.List; public class ExtensionsServiceEndpointProvider { @@ -36,15 +29,4 @@ public String findSelectedEndpoint(InvocableStreamPipesEntity g) throws NoServic ExtensionsServiceEndpointUtils.getPipelineElementType(g)) .getEndpointResourceUrl(); } - - private String getConnectMasterSourcesUrl() throws NoServiceEndpointsAvailableException { - List connectMasterEndpoints = SpServiceDiscovery.getServiceDiscovery() - .getServiceEndpoints(DefaultSpServiceGroups.CORE, true, - Collections.singletonList(DefaultSpServiceTags.CONNECT_MASTER.asString())); - if (connectMasterEndpoints.size() > 0) { - return connectMasterEndpoints.get(0) + GlobalStreamPipesConstants.CONNECT_MASTER_SOURCES_ENDPOINT; - } else { - throw new NoServiceEndpointsAvailableException("Could not find any available connect master service endpoint"); - } - } } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileConstants.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileConstants.java index 7ed4913152..372ddda1d1 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileConstants.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileConstants.java @@ -17,9 +17,14 @@ */ package org.apache.streampipes.manager.file; -import org.apache.streampipes.config.backend.BackendConfig; +import org.apache.streampipes.storage.management.StorageDispatcher; public class FileConstants { - public static final String FILES_BASE_DIR = BackendConfig.INSTANCE.getFilesDir(); + public static final String FILES_BASE_DIR = StorageDispatcher + .INSTANCE + .getNoSqlStore() + .getSpCoreConfigurationStorage() + .get() + .getFilesDir(); } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceHealthCheck.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceHealthCheck.java new file mode 100644 index 0000000000..768430d5a8 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/ServiceHealthCheck.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.health; + +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.storage.api.CRUDStorage; +import org.apache.streampipes.storage.management.StorageDispatcher; + +import org.apache.http.client.fluent.Request; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; + +public class ServiceHealthCheck implements Runnable { + + private static final Logger LOG = LoggerFactory.getLogger(ServiceHealthCheck.class); + + private static final int MAX_UNHEALTHY_DURATION_BEFORE_REMOVAL_MS = 60000; + + private final CRUDStorage storage; + + public ServiceHealthCheck() { + this.storage = StorageDispatcher.INSTANCE.getNoSqlStore().getExtensionsServiceStorage(); + } + + @Override + public void run() { + var registeredServices = getRegisteredServices(); + registeredServices.forEach(this::checkServiceHealth); + } + + private void checkServiceHealth(SpServiceRegistration service) { + String healthCheckUrl = makeHealthCheckUrl(service); + + try { + var response = Request.Get(healthCheckUrl).execute(); + if (response.returnResponse().getStatusLine().getStatusCode() != 200) { + processUnhealthyService(service); + } else { + if (!service.isHealthy()) { + service.setHealthy(true); + updateService(service); + } + } + } catch (IOException e) { + processUnhealthyService(service); + } + } + + private void processUnhealthyService(SpServiceRegistration service) { + if (service.isHealthy()) { + service.setHealthy(false); + service.setFirstTimeSeenUnhealthy(System.currentTimeMillis()); + updateService(service); + } + if (shouldDeleteService(service)) { + LOG.info("Removing service {} which has been unhealthy for more than {} seconds.", + service.getSvcId(), MAX_UNHEALTHY_DURATION_BEFORE_REMOVAL_MS / 1000); + storage.deleteElement(service); + } + } + + private boolean shouldDeleteService(SpServiceRegistration service) { + var currentTimeMillis = System.currentTimeMillis(); + return (currentTimeMillis - service.getFirstTimeSeenUnhealthy() > MAX_UNHEALTHY_DURATION_BEFORE_REMOVAL_MS); + } + + private void updateService(SpServiceRegistration service) { + storage.updateElement(service); + } + + private String makeHealthCheckUrl(SpServiceRegistration service) { + return service.getScheme() + "://" + service.getHost() + ":" + service.getPort() + service.getHealthCheckPath(); + } + + private List getRegisteredServices() { + return storage.getAll(); + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/FormatSelector.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/FormatSelector.java index 128098f7c0..7b38301179 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/FormatSelector.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/FormatSelector.java @@ -18,12 +18,12 @@ package org.apache.streampipes.manager.matching; -import org.apache.streampipes.config.backend.BackendConfig; import org.apache.streampipes.model.SpDataStream; import org.apache.streampipes.model.base.InvocableStreamPipesEntity; import org.apache.streampipes.model.base.NamedStreamPipesEntity; -import org.apache.streampipes.model.config.SpDataFormat; +import org.apache.streampipes.model.configuration.SpDataFormat; import org.apache.streampipes.model.grounding.TransportFormat; +import org.apache.streampipes.storage.management.StorageDispatcher; import org.apache.streampipes.vocabulary.MessageFormat; import java.net.URI; @@ -46,11 +46,17 @@ public TransportFormat getTransportFormat() { .get(0); } else { List prioritizedFormats = - BackendConfig.INSTANCE.getMessagingSettings().getPrioritizedFormats(); + StorageDispatcher + .INSTANCE + .getNoSqlStore() + .getSpCoreConfigurationStorage() + .get() + .getMessagingSettings() + .getPrioritizedFormats(); List supportedFormats = prioritizedFormats .stream() - .filter(pf -> supportsFormat(pf.getMessageFormat())).collect(Collectors.toList()); + .filter(pf -> supportsFormat(pf.getMessageFormat())).toList(); if (supportedFormats.size() > 0) { return new TransportFormat(supportedFormats.get(0).getMessageFormat()); @@ -60,7 +66,7 @@ public TransportFormat getTransportFormat() { } } - public boolean supportsFormat(String format) { + public boolean supportsFormat(String format) { List elements = buildInvocables(); return elements .stream() diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java index 30a152343f..e1eb685f6d 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/ProtocolSelector.java @@ -18,12 +18,12 @@ package org.apache.streampipes.manager.matching; -import org.apache.streampipes.config.backend.BackendConfig; import org.apache.streampipes.manager.util.TopicGenerator; import org.apache.streampipes.model.SpDataStream; import org.apache.streampipes.model.base.InvocableStreamPipesEntity; import org.apache.streampipes.model.base.NamedStreamPipesEntity; -import org.apache.streampipes.model.config.SpProtocol; +import org.apache.streampipes.model.configuration.MessagingSettings; +import org.apache.streampipes.model.configuration.SpProtocol; import org.apache.streampipes.model.grounding.JmsTransportProtocol; import org.apache.streampipes.model.grounding.KafkaTransportProtocol; import org.apache.streampipes.model.grounding.MqttTransportProtocol; @@ -31,6 +31,7 @@ import org.apache.streampipes.model.grounding.PulsarTransportProtocol; import org.apache.streampipes.model.grounding.SimpleTopicDefinition; import org.apache.streampipes.model.grounding.TransportProtocol; +import org.apache.streampipes.storage.management.StorageDispatcher; import java.util.List; import java.util.Set; @@ -40,11 +41,20 @@ public class ProtocolSelector extends GroundingSelector { private final String outputTopic; private final List prioritizedProtocols; + private final MessagingSettings messagingSettings; + public ProtocolSelector(NamedStreamPipesEntity source, Set targets) { super(source, targets); this.outputTopic = TopicGenerator.generateRandomTopic(); + this.messagingSettings = StorageDispatcher + .INSTANCE + .getNoSqlStore() + .getSpCoreConfigurationStorage() + .get() + .getMessagingSettings(); + this.prioritizedProtocols = - BackendConfig.INSTANCE.getMessagingSettings().getPrioritizedProtocols(); + messagingSettings.getPrioritizedProtocols(); } public TransportProtocol getPreferredProtocol() { @@ -68,7 +78,7 @@ && supportsProtocol(NatsTransportProtocol.class)) { return natsTopic(); } else if (prioritizedProtocol.getProtocolClass().equals(PulsarTransportProtocol.class.getCanonicalName()) && supportsProtocol(PulsarTransportProtocol.class)) { - return new PulsarTransportProtocol(BackendConfig.INSTANCE.getPulsarUrl(), + return new PulsarTransportProtocol(messagingSettings.getPulsarUrl(), new SimpleTopicDefinition(outputTopic)); } } @@ -77,30 +87,37 @@ && supportsProtocol(PulsarTransportProtocol.class)) { } private TransportProtocol mqttTopic() { - return new MqttTransportProtocol(BackendConfig.INSTANCE.getMqttHost(), - BackendConfig.INSTANCE.getMqttPort(), - outputTopic); + return new MqttTransportProtocol( + messagingSettings.getMqttHost(), + messagingSettings.getMqttPort(), + outputTopic + ); } private TransportProtocol jmsTopic() { - return new JmsTransportProtocol(BackendConfig.INSTANCE.getJmsHost(), - BackendConfig.INSTANCE.getJmsPort(), - outputTopic); + return new JmsTransportProtocol( + messagingSettings.getJmsHost(), + messagingSettings.getJmsPort(), + outputTopic + ); } private TransportProtocol natsTopic() { return new NatsTransportProtocol( - BackendConfig.INSTANCE.getNatsHost(), - BackendConfig.INSTANCE.getNatsPort(), - outputTopic); + messagingSettings.getNatsHost(), + messagingSettings.getNatsPort(), + outputTopic + ); } private TransportProtocol kafkaTopic() { - return new KafkaTransportProtocol(BackendConfig.INSTANCE.getKafkaHost(), - BackendConfig.INSTANCE.getKafkaPort(), + return new KafkaTransportProtocol( + messagingSettings.getKafkaHost(), + messagingSettings.getKafkaPort(), outputTopic, - BackendConfig.INSTANCE.getZookeeperHost(), - BackendConfig.INSTANCE.getZookeeperPort()); + messagingSettings.getZookeeperHost(), + messagingSettings.getZookeeperPort() + ); } @@ -114,6 +131,5 @@ public boolean supportsProtocol(Class protocol) .getTransportProtocols() .stream() .anyMatch(protocol::isInstance)); - } } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/monitoring/pipeline/ExtensionsServiceLogExecutor.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/monitoring/pipeline/ExtensionsServiceLogExecutor.java index 83eba204b4..ee8e19885f 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/monitoring/pipeline/ExtensionsServiceLogExecutor.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/monitoring/pipeline/ExtensionsServiceLogExecutor.java @@ -25,8 +25,8 @@ import org.apache.streampipes.resource.management.SpResourceManager; import org.apache.streampipes.serializers.json.JacksonSerializer; import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceGroups; import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTags; +import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTypes; import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.http.client.fluent.Request; @@ -71,7 +71,7 @@ private Principal getServiceAdmin() { private List getActiveExtensionsEndpoints() { return SpServiceDiscovery.getServiceDiscovery().getServiceEndpoints( - DefaultSpServiceGroups.EXT, + DefaultSpServiceTypes.EXT, true, List.of(DefaultSpServiceTags.PE.asString(), DefaultSpServiceTags.CONNECT_WORKER.asString()) ); diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/AutoInstallation.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/AutoInstallation.java index 0cad980a59..5782317016 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/AutoInstallation.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/AutoInstallation.java @@ -27,13 +27,18 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; -public class AutoInstallation { +public class AutoInstallation implements BackgroundTaskNotifier { private static final Logger LOG = LoggerFactory.getLogger(AutoInstallation.class); - private Environment env; + private final Environment env; + private final AtomicInteger errorCount = new AtomicInteger(); + private int numberOfBackgroundSteps = 0; + private int executedBackgroundSteps = 0; public AutoInstallation() { this.env = Environments.getEnvironment(); @@ -43,19 +48,22 @@ public void startAutoInstallation() { InitialSettings settings = collectInitialSettings(); List steps = InstallationConfiguration.getInstallationSteps(settings); - AtomicInteger errorCount = new AtomicInteger(); + List backgroundSteps = InstallationConfiguration.getBackgroundInstallationSteps(settings, this); + numberOfBackgroundSteps = backgroundSteps.size(); steps.forEach(step -> { step.install(); errorCount.addAndGet(step.getErrorCount()); }); - if (errorCount.get() > 0) { - LOG.error("{} errors occurred during the setup process", errorCount); - } else { - BackendConfig.INSTANCE.setIsConfigured(true); - LOG.info("Initial setup completed successfully - you can now open the login page in the browser."); - } - LOG.info("\n\n**********\n\nAuto-Setup finished\n\n**********\n\n"); + + initBackgroundJobs(backgroundSteps); + } + + private void initBackgroundJobs(List backgroundSteps) { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + + backgroundSteps.forEach(executorService::execute); + executorService.shutdown(); } private InitialSettings collectInitialSettings() { @@ -103,4 +111,20 @@ private String getStringOrDefault(StringEnvironmentVariable variable) { return variable.getDefault(); } } + + @Override + public void notifyFinished(int ec) { + errorCount.addAndGet(ec); + executedBackgroundSteps++; + + if (executedBackgroundSteps == numberOfBackgroundSteps) { + if (errorCount.get() > 0) { + LOG.error("{} errors occurred during the setup process", errorCount); + } else { + BackendConfig.INSTANCE.setIsConfigured(true); + LOG.info("Initial setup completed successfully - you can now open the login page in the browser."); + } + LOG.info("\n\n**********\n\nAuto-Setup finished\n\n**********\n\n"); + } + } } diff --git a/ui/src/app/configuration/consul-configs-number/consul-configs-number.component.css b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/BackgroundTaskNotifier.java similarity index 85% rename from ui/src/app/configuration/consul-configs-number/consul-configs-number.component.css rename to streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/BackgroundTaskNotifier.java index 13cbc4aacb..28083e9404 100644 --- a/ui/src/app/configuration/consul-configs-number/consul-configs-number.component.css +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/BackgroundTaskNotifier.java @@ -15,3 +15,11 @@ * limitations under the License. * */ + +package org.apache.streampipes.manager.setup; + +public interface BackgroundTaskNotifier { + + void notifyFinished(int errorCount); + +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/ExtensionsInstallationTask.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/ExtensionsInstallationTask.java new file mode 100644 index 0000000000..51864794e4 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/ExtensionsInstallationTask.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.setup; + +import org.apache.streampipes.manager.endpoint.EndpointFetcher; +import org.apache.streampipes.model.client.endpoint.ExtensionsServiceEndpoint; +import org.apache.streampipes.model.client.setup.InitialSettings; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class ExtensionsInstallationTask implements Runnable { + + private static final Logger LOG = LoggerFactory.getLogger(ExtensionsInstallationTask.class); + + private static final int MAX_RETRIES = 4; + private static final int SLEEP_TIME_SECONDS = 3; + + private final InitialSettings settings; + private final BackgroundTaskNotifier callback; + + public ExtensionsInstallationTask(InitialSettings settings, + BackgroundTaskNotifier callback) { + this.settings = settings; + this.callback = callback; + } + + @Override + public void run() { + List steps = new ArrayList<>(); + if (settings.getInstallPipelineElements()) { + List endpoints; + int numberOfAttempts = 0; + do { + endpoints = new EndpointFetcher().getEndpoints(); + numberOfAttempts++; + if (endpoints.size() == 0) { + LOG.info("Found 0 endpoints - waiting {} seconds to make sure all endpoints have properly started", + SLEEP_TIME_SECONDS); + try { + TimeUnit.SECONDS.sleep(SLEEP_TIME_SECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } while (endpoints.size() == 0 && numberOfAttempts < MAX_RETRIES); + LOG.info("Found {} endpoints from which we will install extensions.", endpoints.size()); + LOG.info( + "Further available extensions can always be installed by navigating to the 'Install pipeline elements' view"); + for (ExtensionsServiceEndpoint endpoint : endpoints) { + steps.add(new PipelineElementInstallationStep(endpoint, settings.getInitialAdminUserSid())); + } + + AtomicInteger errorCount = new AtomicInteger(0); + steps.forEach(step -> { + step.install(); + errorCount.addAndGet(step.getErrorCount()); + }); + callback.notifyFinished(errorCount.get()); + } + } +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/InstallationConfiguration.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/InstallationConfiguration.java index aab4bc37f6..8a067836ac 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/InstallationConfiguration.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/InstallationConfiguration.java @@ -18,26 +18,17 @@ package org.apache.streampipes.manager.setup; -import org.apache.streampipes.manager.endpoint.EndpointFetcher; -import org.apache.streampipes.model.client.endpoint.ExtensionsServiceEndpoint; import org.apache.streampipes.model.client.setup.InitialSettings; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; public class InstallationConfiguration { - private static final Logger LOG = LoggerFactory.getLogger(InstallationConfiguration.class); - private static final int MAX_RETRIES = 3; - private static final int SLEEP_TIME_SECONDS = 5; - public static List getInstallationSteps(InitialSettings settings) { List steps = new ArrayList<>(); + steps.add(new SpCoreConfigurationStep()); steps.add(new CouchDbInstallationStep()); steps.add(new UserRegistrationInstallationStep( settings.getAdminEmail(), @@ -46,30 +37,11 @@ public static List getInstallationSteps(InitialSettings settin settings.getInitialServiceAccountSecret(), settings.getInitialAdminUserSid())); - if (settings.getInstallPipelineElements()) { - List endpoints; - int numberOfAttempts = 0; - do { - endpoints = new EndpointFetcher().getEndpoints(); - numberOfAttempts++; - if (endpoints.size() == 0) { - LOG.info("Found 0 endpoints - waiting {} seconds to make sure all endpoints have properly started", - SLEEP_TIME_SECONDS); - try { - TimeUnit.SECONDS.sleep(SLEEP_TIME_SECONDS); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } while (endpoints.size() == 0 && numberOfAttempts < MAX_RETRIES); - LOG.info("Found {} endpoints from which we will install extensions.", endpoints.size()); - LOG.info( - "Further available extensions can always be installed by navigating to the 'Install pipeline elements' view"); - for (ExtensionsServiceEndpoint endpoint : endpoints) { - steps.add(new PipelineElementInstallationStep(endpoint, settings.getInitialAdminUserSid())); - } - } - return steps; } + + public static List getBackgroundInstallationSteps(InitialSettings settings, + BackgroundTaskNotifier callback) { + return List.of(new ExtensionsInstallationTask(settings, callback)); + } } diff --git a/ui/src/app/configuration/consul-service/consul-service.component.css b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/SpCoreConfigurationStep.java similarity index 56% rename from ui/src/app/configuration/consul-service/consul-service.component.css rename to streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/SpCoreConfigurationStep.java index 1bcf4ab7e8..187d677efc 100644 --- a/ui/src/app/configuration/consul-service/consul-service.component.css +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/SpCoreConfigurationStep.java @@ -1,49 +1,37 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -.service-icon-passing { - cursor: default; - color: #4caf50; -} - -.service-icon-critical { - cursor: default; - color: #f44336; -} - -.show-configuration-icon { - cursor: pointer; - color: rgba(0, 0, 0, 0.54); -} - -.tooltip-icon { - cursor: default; - font-size: 150%; -} - -.submit-button { - width: 100%; -} - -.form-field { - width: 100%; -} - -.checkbox-field { - padding-bottom: 1.25em; -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.setup; + +import org.apache.streampipes.model.configuration.DefaultSpCoreConfiguration; +import org.apache.streampipes.storage.management.StorageDispatcher; + +public class SpCoreConfigurationStep extends InstallationStep { + @Override + public void install() { + var coreCfg = new DefaultSpCoreConfiguration().make(); + + StorageDispatcher.INSTANCE.getNoSqlStore().getSpCoreConfigurationStorage().createElement(coreCfg); + new StreamPipesEnvChecker().updateEnvironmentVariables(); + } + + @Override + public String getTitle() { + return "Storing default configuration to database"; + } +} diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesEnvChecker.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/StreamPipesEnvChecker.java similarity index 78% rename from streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesEnvChecker.java rename to streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/StreamPipesEnvChecker.java index 64e187c78a..17dc5072e8 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesEnvChecker.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/setup/StreamPipesEnvChecker.java @@ -16,13 +16,15 @@ * */ -package org.apache.streampipes.service.core; +package org.apache.streampipes.manager.setup; import org.apache.streampipes.commons.environment.Environment; import org.apache.streampipes.commons.environment.Environments; -import org.apache.streampipes.config.backend.BackendConfig; -import org.apache.streampipes.config.backend.model.JwtSigningMode; -import org.apache.streampipes.config.backend.model.LocalAuthConfig; +import org.apache.streampipes.model.configuration.JwtSigningMode; +import org.apache.streampipes.model.configuration.LocalAuthConfig; +import org.apache.streampipes.model.configuration.SpCoreConfiguration; +import org.apache.streampipes.storage.api.ISpCoreConfigurationStorage; +import org.apache.streampipes.storage.management.StorageDispatcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,19 +37,27 @@ public class StreamPipesEnvChecker { private static final Logger LOG = LoggerFactory.getLogger(StreamPipesEnvChecker.class); - BackendConfig coreConfig; + private ISpCoreConfigurationStorage configStorage; + private SpCoreConfiguration coreConfig; - private Environment env; + private final Environment env; public StreamPipesEnvChecker() { this.env = Environments.getEnvironment(); } public void updateEnvironmentVariables() { - this.coreConfig = BackendConfig.INSTANCE; + this.configStorage = StorageDispatcher + .INSTANCE + .getNoSqlStore() + .getSpCoreConfigurationStorage(); - LOG.info("Checking and updating environment variables..."); - updateJwtSettings(); + if (configStorage.getAll().size() > 0) { + this.coreConfig = configStorage.get(); + + LOG.info("Checking and updating environment variables..."); + updateJwtSettings(); + } } private void updateJwtSettings() { @@ -92,7 +102,8 @@ private void updateJwtSettings() { } if (!incompleteConfig) { LOG.info("Updating local auth config with signing mode {}", localAuthConfig.getJwtSigningMode().name()); - coreConfig.updateLocalAuthConfig(localAuthConfig); + coreConfig.setLocalAuthConfig(localAuthConfig); + configStorage.updateElement(coreConfig); } } diff --git a/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/AbstractRestResource.java b/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/AbstractRestResource.java index 10c74f6192..b231450db9 100644 --- a/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/AbstractRestResource.java +++ b/streampipes-rest-core-base/src/main/java/org/apache/streampipes/rest/core/base/impl/AbstractRestResource.java @@ -32,13 +32,11 @@ import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorageCache; import org.apache.streampipes.storage.api.IPipelineElementTemplateStorage; import org.apache.streampipes.storage.api.IPipelineStorage; +import org.apache.streampipes.storage.api.ISpCoreConfigurationStorage; import org.apache.streampipes.storage.api.IUserStorage; import org.apache.streampipes.storage.api.IVisualizationStorage; import org.apache.streampipes.storage.management.StorageDispatcher; import org.apache.streampipes.storage.management.StorageManager; -import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.ISpKvManagement; -import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery; import org.apache.http.client.ClientProtocolException; @@ -51,6 +49,10 @@ public abstract class AbstractRestResource extends AbstractSharedRestInterface { + protected ISpCoreConfigurationStorage getSpCoreConfigurationStorage() { + return getNoSqlStorage().getSpCoreConfigurationStorage(); + } + protected IPipelineElementDescriptionStorageCache getPipelineElementRdfStorage() { return StorageManager.INSTANCE.getPipelineElementStorage(); } @@ -110,19 +112,6 @@ protected Response constructErrorMessage(Notification... notifications) { return statusMessage(new ErrorMessage(notifications)); } -// protected String getCurrentUsername() throws AuthenticationException { -// if (SecurityContext.getSubject().isAuthenticated()) { -// return SecurityUtils.getSubject().getPrincipal().toString(); -// } -// throw new AuthenticationException("Not authenticated"); -// } - - protected boolean authorized(String username) { - // TODO SEC - //return username.equals(SecurityUtils.getSubject().getPrincipal().toString()); - return true; - } - @SuppressWarnings("deprecation") protected String decode(String encodedString) { return URLDecoder.decode(encodedString); @@ -145,14 +134,6 @@ protected Response unauthorized() { return Response.status(Response.Status.UNAUTHORIZED).build(); } - protected ISpKvManagement getKeyValueStore() { - return SpServiceDiscovery.getKeyValueStore(); - } - - protected ISpServiceDiscovery getServiceDiscovery() { - return SpServiceDiscovery.getServiceDiscovery(); - } - protected SpResourceManager getSpResourceManager() { return new SpResourceManager(); } diff --git a/streampipes-rest-shared/src/main/java/org/apache/streampipes/rest/shared/impl/AbstractSharedRestInterface.java b/streampipes-rest-shared/src/main/java/org/apache/streampipes/rest/shared/impl/AbstractSharedRestInterface.java index 80a3d4ecb1..2f7211807e 100644 --- a/streampipes-rest-shared/src/main/java/org/apache/streampipes/rest/shared/impl/AbstractSharedRestInterface.java +++ b/streampipes-rest-shared/src/main/java/org/apache/streampipes/rest/shared/impl/AbstractSharedRestInterface.java @@ -58,6 +58,10 @@ protected Response ok() { return Response.ok().build(); } + protected Response created() { + return Response.status(201).build(); + } + protected Response fail() { return Response.serverError().build(); } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/Authentication.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/Authentication.java index 5d7a5e1516..ad0dfc6f1f 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/Authentication.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/Authentication.java @@ -20,13 +20,12 @@ import org.apache.streampipes.commons.exceptions.UserNotFoundException; import org.apache.streampipes.commons.exceptions.UsernameAlreadyTakenException; -import org.apache.streampipes.config.backend.BackendConfig; -import org.apache.streampipes.config.backend.model.GeneralConfig; import org.apache.streampipes.model.client.user.JwtAuthenticationResponse; import org.apache.streampipes.model.client.user.LoginRequest; import org.apache.streampipes.model.client.user.Principal; import org.apache.streampipes.model.client.user.RegistrationData; import org.apache.streampipes.model.client.user.UserAccount; +import org.apache.streampipes.model.configuration.GeneralConfig; import org.apache.streampipes.model.message.ErrorMessage; import org.apache.streampipes.model.message.NotificationType; import org.apache.streampipes.model.message.Notifications; @@ -97,7 +96,7 @@ public Response doLogin() { @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public Response doRegister(RegistrationData data) { - GeneralConfig config = BackendConfig.INSTANCE.getGeneralConfig(); + GeneralConfig config = getSpCoreConfigurationStorage().get().getGeneralConfig(); if (!config.isAllowSelfRegistration()) { throw new WebApplicationException(Response.Status.FORBIDDEN); } @@ -129,7 +128,7 @@ public Response sendPasswordRecoveryLink(@PathParam("username") String username) @JacksonSerialized @Produces(MediaType.APPLICATION_JSON) public Response getAuthSettings() { - GeneralConfig config = BackendConfig.INSTANCE.getGeneralConfig(); + GeneralConfig config = getSpCoreConfigurationStorage().get().getGeneralConfig(); Map response = new HashMap<>(); response.put("allowSelfRegistration", config.isAllowSelfRegistration()); response.put("allowPasswordRecovery", config.isAllowPasswordRecovery()); diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/EmailResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/EmailResource.java index 1ebbbecce7..84d80ce886 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/EmailResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/EmailResource.java @@ -17,7 +17,6 @@ */ package org.apache.streampipes.rest.impl; -import org.apache.streampipes.config.backend.BackendConfig; import org.apache.streampipes.mail.MailSender; import org.apache.streampipes.model.mail.SpEmail; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; @@ -35,7 +34,7 @@ public class EmailResource extends AbstractAuthGuardedRestResource { @POST @Consumes(MediaType.APPLICATION_JSON) public Response sendEmail(SpEmail email) { - if (BackendConfig.INSTANCE.getEmailConfig().isEmailConfigured()) { + if (getSpCoreConfigurationStorage().get().getEmailConfig().isEmailConfigured()) { new MailSender().sendEmail(email); return ok(); } else { diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineResource.java index ac331df2ab..b42f9e02d0 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineResource.java @@ -235,6 +235,7 @@ public Response overwritePipeline(@PathParam("pipelineId") String pipelineId, Pi } storedPipeline.setCreatedAt(System.currentTimeMillis()); storedPipeline.setPipelineCategories(pipeline.getPipelineCategories()); + storedPipeline.setHealthStatus(pipeline.getHealthStatus()); storedPipeline.setPipelineNotifications(pipeline.getPipelineNotifications()); Operations.updatePipeline(storedPipeline); SuccessMessage message = Notifications.success("Pipeline modified"); diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ConsulConfig.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ConsulConfig.java deleted file mode 100644 index 17bdb22a84..0000000000 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ConsulConfig.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.rest.impl.admin; - - -import org.apache.streampipes.config.backend.BackendConfig; -import org.apache.streampipes.model.config.MessagingSettings; -import org.apache.streampipes.rest.core.base.impl.AbstractRestResource; -import org.apache.streampipes.rest.security.AuthConstants; -import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; -import org.apache.streampipes.svcdiscovery.api.ISpKvManagement; -import org.apache.streampipes.svcdiscovery.api.model.ConfigItem; -import org.apache.streampipes.svcdiscovery.api.model.PeConfig; -import org.apache.streampipes.svcdiscovery.consul.ConsulSpConfig; - -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.stereotype.Component; - -import jakarta.ws.rs.Consumes; -import jakarta.ws.rs.DELETE; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; - -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -@Path("/v2/consul") -@Component -public class ConsulConfig extends AbstractRestResource { - - private static final Logger LOG = LoggerFactory.getLogger(ConsulConfig.class); - - @GET - @Produces(MediaType.APPLICATION_JSON) - @JacksonSerialized - @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) - public Response getAllServiceConfigs() { - LOG.info("Request for all service configs"); - Map peServices = getServiceDiscovery().getExtensionsServiceGroups(); - - List peConfigs = new LinkedList<>(); - - for (Map.Entry entry : peServices.entrySet()) { - String serviceStatus = entry.getValue(); - String mainKey = ConsulSpConfig.SERVICE_ROUTE_PREFIX + entry.getKey(); - - Map meta = new HashMap<>(); - meta.put("status", serviceStatus); - List configItems = getConfigForService(entry.getKey()); - - PeConfig peConfig = new PeConfig(); - - for (ConfigItem configItem : configItems) { - if (configItem.getKey().endsWith("SP_SERVICE_NAME")) { - configItems.remove(configItem); - peConfig.setName(configItem.getValue()); - break; - } - } - - peConfig.setMeta(meta); - peConfig.setMainKey(mainKey); - peConfig.setConfigs(configItems); - - peConfigs.add(peConfig); - } - - return ok(peConfigs); - } - - @POST - @Produces(MediaType.APPLICATION_JSON) - @JacksonSerialized - @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) - public Response saveServiceConfig(PeConfig peConfig) { - - ISpKvManagement keyValueStore = getKeyValueStore(); - LOG.info("Request to update a service config"); - for (ConfigItem configItem : peConfig.getConfigs()) { - String value = configItem.getValue(); - switch (configItem.getValueType()) { - case "xs:boolean": - if (!("true".equals(value) || "false".equals(value))) { - LOG.error(value + " is not from the type: xs:boolean"); - return Response.status(Response.Status.BAD_REQUEST) - .entity(value + " is not from the type: xs:boolean").build(); - } - break; - case "xs:integer": - try { - Integer.valueOf(value); - } catch (java.lang.NumberFormatException e) { - LOG.error(value + " is not from the type: xs:integer"); - return Response.status(Response.Status.BAD_REQUEST) - .entity(value + " is not from the type: xs:integer").build(); - } - break; - case "xs:double": - try { - Double.valueOf(value); - } catch (java.lang.NumberFormatException e) { - LOG.error(value + " is not from the type: xs:double"); - return Response.status(Response.Status.BAD_REQUEST) - .entity(value + " is not from the type: xs:double").build(); - } - break; - case "xs:string": - break; - default: - LOG.error(configItem.getValueType() + " is not a supported type"); - return Response.status(Response.Status.BAD_REQUEST) - .entity(configItem.getValueType() + " is not a supported type").build(); - } - } - - String prefix = peConfig.getMainKey(); - - for (ConfigItem configItem : peConfig.getConfigs()) { - JsonObject jsonObj = new Gson().toJsonTree(configItem).getAsJsonObject(); - jsonObj.entrySet().removeIf(e -> e.getKey().equals("key")); - keyValueStore.updateConfig(configItem.getKey(), jsonObj.toString(), - configItem.isPassword()); - } - return Response.status(Response.Status.OK).build(); - } - - @DELETE - @Produces(MediaType.APPLICATION_JSON) - @JacksonSerialized - @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) - public Response deleteService(String serviceName) { - LOG.info("Request to delete a service config"); - getServiceDiscovery().deregisterService(serviceName); - return Response.status(Response.Status.OK).build(); - } - - @GET - @Produces(MediaType.APPLICATION_JSON) - @JacksonSerialized - @Path("/messaging") - @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) - public Response getMessagingSettings() { - return ok(BackendConfig.INSTANCE.getMessagingSettings()); - } - - @POST - @Produces(MediaType.APPLICATION_JSON) - @Consumes(MediaType.APPLICATION_JSON) - @JacksonSerialized - @Path("messaging") - @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) - public Response updateMessagingSettings(MessagingSettings messagingSettings) { - BackendConfig.INSTANCE.setMessagingSettings(messagingSettings); - return ok(); - } - - public List getConfigForService(String serviceId) { - Map keyValues = getKeyValueStore().getKeyValue(ConsulSpConfig.SERVICE_ROUTE_PREFIX + serviceId); - - List configItems = new LinkedList<>(); - - for (Map.Entry entry : keyValues.entrySet()) { - ConfigItem configItem = new Gson().fromJson(entry.getValue(), ConfigItem.class); - configItem.setKey(entry.getKey()); - - configItems.add(configItem); - } - return configItems; - } - - -} diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/EmailConfigurationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/EmailConfigurationResource.java index a5c18094fb..a14e8f7bcf 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/EmailConfigurationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/EmailConfigurationResource.java @@ -17,9 +17,8 @@ */ package org.apache.streampipes.rest.impl.admin; -import org.apache.streampipes.config.backend.BackendConfig; -import org.apache.streampipes.config.backend.model.EmailConfig; import org.apache.streampipes.mail.MailTester; +import org.apache.streampipes.model.configuration.EmailConfig; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; import org.apache.streampipes.rest.security.AuthConstants; import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; @@ -49,7 +48,7 @@ public class EmailConfigurationResource extends AbstractAuthGuardedRestResource @Produces(MediaType.APPLICATION_JSON) @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) public Response getMailConfiguration() { - return ok(BackendConfig.INSTANCE.getEmailConfig()); + return ok(getSpCoreConfigurationStorage().get().getEmailConfig()); } @PUT @@ -67,7 +66,10 @@ public Response updateMailConfiguration(EmailConfig config) { config.setSmtpPassword(SecretEncryptionManager.encrypt(config.getSmtpPassword())); config.setSmtpPassEncrypted(true); } - BackendConfig.INSTANCE.updateEmailConfig(config); + var storage = getSpCoreConfigurationStorage(); + var cfg = storage.get(); + cfg.setEmailConfig(config); + storage.updateElement(cfg); return ok(); } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/GeneralConfigurationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/GeneralConfigurationResource.java index 4b32fc1477..df1a06a051 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/GeneralConfigurationResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/GeneralConfigurationResource.java @@ -17,8 +17,7 @@ */ package org.apache.streampipes.rest.impl.admin; -import org.apache.streampipes.config.backend.BackendConfig; -import org.apache.streampipes.config.backend.model.GeneralConfig; +import org.apache.streampipes.model.configuration.GeneralConfig; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; import org.apache.streampipes.rest.security.AuthConstants; import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; @@ -51,7 +50,7 @@ public class GeneralConfigurationResource extends AbstractAuthGuardedRestResourc @Produces(MediaType.APPLICATION_JSON) @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) public GeneralConfig getGeneralConfiguration() { - return BackendConfig.INSTANCE.getGeneralConfig(); + return getSpCoreConfigurationStorage().get().getGeneralConfig(); } @PUT @@ -60,7 +59,10 @@ public GeneralConfig getGeneralConfiguration() { @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) public Response updateGeneralConfiguration(GeneralConfig config) { config.setConfigured(true); - BackendConfig.INSTANCE.updateGeneralConfig(config); + var storage = getSpCoreConfigurationStorage(); + var cfg = storage.get(); + cfg.setGeneralConfig(config); + storage.updateElement(cfg); return ok(); } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MessagingConfigurationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MessagingConfigurationResource.java new file mode 100644 index 0000000000..ed421d5cd2 --- /dev/null +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/MessagingConfigurationResource.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.rest.impl.admin; + +import org.apache.streampipes.model.configuration.MessagingSettings; +import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; +import org.apache.streampipes.rest.security.AuthConstants; +import org.apache.streampipes.rest.shared.annotation.JacksonSerialized; + +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.stereotype.Component; + +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +@Path("/v2/messaging") +@Component +public class MessagingConfigurationResource extends AbstractAuthGuardedRestResource { + + @GET + @Produces(MediaType.APPLICATION_JSON) + @JacksonSerialized + @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) + public Response getMessagingSettings() { + return ok(getSpCoreConfigurationStorage().get().getMessagingSettings()); + } + + @POST + @Produces(MediaType.APPLICATION_JSON) + @Consumes(MediaType.APPLICATION_JSON) + @JacksonSerialized + @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) + public Response updateMessagingSettings(MessagingSettings messagingSettings) { + var cfg = getSpCoreConfigurationStorage().get(); + cfg.setMessagingSettings(messagingSettings); + getSpCoreConfigurationStorage().updateElement(cfg); + return ok(); + } +} diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceConfigurationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceConfigurationResource.java new file mode 100644 index 0000000000..8a89b41dbe --- /dev/null +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceConfigurationResource.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.rest.impl.admin; + +import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; +import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; +import org.apache.streampipes.rest.security.AuthConstants; +import org.apache.streampipes.storage.api.CRUDStorage; + +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.stereotype.Component; + +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +@Path("/v2/extensions-services-configurations") +@Component +@PreAuthorize(AuthConstants.IS_ADMIN_ROLE) +public class ServiceConfigurationResource extends AbstractAuthGuardedRestResource { + + private final CRUDStorage extensionsServicesConfigStorage = + getNoSqlStorage().getExtensionsServiceConfigurationStorage(); + + + @GET + @Produces(MediaType.APPLICATION_JSON) + public Response getAllServiceConfigurations() { + return ok(extensionsServicesConfigStorage.getAll()); + } + + @GET + @Path("{serviceGroup}") + @Produces(MediaType.APPLICATION_JSON) + public Response getServiceConfiguration(@PathParam("serviceGroup") String serviceGroup) { + var config = extensionsServicesConfigStorage.getElementById(serviceGroup); + return config != null ? ok(config) : notFound(); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + public Response registerServiceConfiguration(SpServiceConfiguration serviceConfiguration) { + if (extensionsServicesConfigStorage.getElementById(serviceConfiguration.getServiceGroup()) == null) { + extensionsServicesConfigStorage.createElement(serviceConfiguration); + return created(); + } else { + return ok(); + } + } + + @PUT + @Path("{serviceGroup}") + @Consumes(MediaType.APPLICATION_JSON) + public Response registerServiceConfiguration(@PathParam("serviceGroup") String serviceGroup, + SpServiceConfiguration serviceConfiguration) { + if (extensionsServicesConfigStorage.getElementById(serviceGroup) != null) { + extensionsServicesConfigStorage.updateElement(serviceConfiguration); + return ok(); + } else { + return badRequest(); + } + } +} diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceRegistrationResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceRegistrationResource.java new file mode 100644 index 0000000000..f89cf34ace --- /dev/null +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ServiceRegistrationResource.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.rest.impl.admin; + +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; +import org.apache.streampipes.rest.security.AuthConstants; +import org.apache.streampipes.storage.api.CRUDStorage; + +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.stereotype.Component; + +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +@Path("/v2/extensions-services") +@Component +@PreAuthorize(AuthConstants.IS_ADMIN_ROLE) +public class ServiceRegistrationResource extends AbstractAuthGuardedRestResource { + + private final CRUDStorage extensionsServiceStorage = + getNoSqlStorage().getExtensionsServiceStorage(); + + @GET + @Produces(MediaType.APPLICATION_JSON) + public Response getRegisteredServices() { + return ok(extensionsServiceStorage.getAll()); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + public Response registerService(SpServiceRegistration serviceRegistration) { + extensionsServiceStorage.createElement(serviceRegistration); + return ok(); + } + + @POST + @Path("/{serviceId}") + public Response unregisterService(@PathParam("serviceId") String serviceId) { + try { + var serviceRegistration = extensionsServiceStorage.getElementById(serviceId); + extensionsServiceStorage.deleteElement(serviceRegistration); + return ok(); + } catch (IllegalArgumentException e) { + return badRequest("Could not find registered service with id " + serviceId); + } + } +} diff --git a/streampipes-service-base/src/main/java/org/apache/streampipes/service/base/StreamPipesServiceBase.java b/streampipes-service-base/src/main/java/org/apache/streampipes/service/base/StreamPipesServiceBase.java index 5b3a5f84c3..0d516c3a8d 100644 --- a/streampipes-service-base/src/main/java/org/apache/streampipes/service/base/StreamPipesServiceBase.java +++ b/streampipes-service-base/src/main/java/org/apache/streampipes/service/base/StreamPipesServiceBase.java @@ -17,29 +17,18 @@ */ package org.apache.streampipes.service.base; -import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceRegistrationRequest; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceTag; - import org.apache.commons.lang3.RandomStringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import java.net.UnknownHostException; import java.util.Collections; -import java.util.List; public abstract class StreamPipesServiceBase { public static final String AUTO_GENERATED_SERVICE_ID = RandomStringUtils.randomAlphanumeric(6); - private static final Logger LOG = LoggerFactory.getLogger(StreamPipesServiceBase.class); public void startStreamPipesService(Class serviceClass, - String serviceGroup, - String serviceId, BaseNetworkingConfig networkingConfig) throws UnknownHostException { - registerService(serviceGroup, serviceId, networkingConfig); runApplication(serviceClass, networkingConfig.getPort()); } @@ -50,29 +39,6 @@ private void runApplication(Class serviceClass, app.run(); } - private void registerService(String serviceGroup, - String serviceId, - BaseNetworkingConfig networkingConfig) { - SpServiceRegistrationRequest req = SpServiceRegistrationRequest.from( - serviceGroup, - serviceId, - networkingConfig.getHost(), - networkingConfig.getPort(), - getServiceTags(), - getHealthCheckPath()); - - SpServiceDiscovery - .getServiceDiscovery() - .registerService(req); - } - - protected abstract List getServiceTags(); - - protected void deregisterService(String serviceId) { - LOG.info("Deregistering service (id={})...", serviceId); - SpServiceDiscovery.getServiceDiscovery().deregisterService(serviceId); - } - protected String getHealthCheckPath() { return "/svchealth/" + AUTO_GENERATED_SERVICE_ID; } diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java index 295f60bd93..6e27b67685 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesCoreApplication.java @@ -19,9 +19,11 @@ import org.apache.streampipes.config.backend.BackendConfig; import org.apache.streampipes.manager.health.PipelineHealthCheck; +import org.apache.streampipes.manager.health.ServiceHealthCheck; import org.apache.streampipes.manager.monitoring.pipeline.ExtensionsServiceLogExecutor; import org.apache.streampipes.manager.operations.Operations; import org.apache.streampipes.manager.setup.AutoInstallation; +import org.apache.streampipes.manager.setup.StreamPipesEnvChecker; import org.apache.streampipes.messaging.SpProtocolManager; import org.apache.streampipes.messaging.jms.SpJmsProtocolFactory; import org.apache.streampipes.messaging.kafka.SpKafkaProtocolFactory; @@ -37,9 +39,6 @@ import org.apache.streampipes.storage.api.IPipelineStorage; import org.apache.streampipes.storage.couchdb.utils.CouchDbViewGenerator; import org.apache.streampipes.storage.management.StorageDispatcher; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceGroups; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTags; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceTag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,7 +51,6 @@ import jakarta.annotation.PreDestroy; import java.net.UnknownHostException; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -82,8 +80,12 @@ public class StreamPipesCoreApplication extends StreamPipesServiceBase { private static final int HEALTH_CHECK_INTERVAL = 60; private static final TimeUnit HEALTH_CHECK_UNIT = TimeUnit.SECONDS; + private static final int SERVICE_HEALTH_CHECK_INTERVAL = 60; + private static final TimeUnit SERVICE_HEALTH_CHECK_UNIT = TimeUnit.SECONDS; + private ScheduledExecutorService executorService; private ScheduledExecutorService healthCheckExecutorService; + private ScheduledExecutorService serviceHealthCheckExecutorService; private ScheduledExecutorService logCheckExecutorService; private final Map failedPipelines = new HashMap<>(); @@ -104,8 +106,6 @@ public void initialize(SupportedProtocols supportedProtocols) { registerProtocols(supportedProtocols); BaseNetworkingConfig networkingConfig = BaseNetworkingConfig.defaultResolution(8030); startStreamPipesService(StreamPipesCoreApplication.class, - DefaultSpServiceGroups.CORE, - serviceId(), networkingConfig); } catch (UnknownHostException e) { LOG.error( @@ -118,26 +118,30 @@ protected void registerProtocols(SupportedProtocols protocols) { protocols.getSupportedProtocols().forEach(SpProtocolManager.INSTANCE::register); } - private String serviceId() { - return DefaultSpServiceGroups.CORE + "-" + AUTO_GENERATED_SERVICE_ID; - } - @PostConstruct public void init() { this.executorService = Executors.newSingleThreadScheduledExecutor(); this.healthCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); this.logCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); + this.serviceHealthCheckExecutorService = Executors.newSingleThreadScheduledExecutor(); new StreamPipesEnvChecker().updateEnvironmentVariables(); new CouchDbViewGenerator().createGenericDatabaseIfNotExists(); if (!isConfigured()) { doInitialSetup(); + } else { + new MigrationsHandler().performMigrations(); } - new MigrationsHandler().performMigrations(); + executorService.schedule(this::startAllPreviouslyStoppedPipelines, 10, TimeUnit.SECONDS); + + LOG.info("Service health check will run every {} seconds", SERVICE_HEALTH_CHECK_INTERVAL); + serviceHealthCheckExecutorService.scheduleAtFixedRate(new ServiceHealthCheck(), + SERVICE_HEALTH_CHECK_INTERVAL, + SERVICE_HEALTH_CHECK_INTERVAL, + SERVICE_HEALTH_CHECK_UNIT); - executorService.schedule(this::startAllPreviouslyStoppedPipelines, 5, TimeUnit.SECONDS); LOG.info("Pipeline health check will run every {} seconds", HEALTH_CHECK_INTERVAL); healthCheckExecutorService.scheduleAtFixedRate(new PipelineHealthCheck(), HEALTH_CHECK_INTERVAL, @@ -149,8 +153,6 @@ public void init() { LOG_FETCH_INTERVAL, LOG_FETCH_INTERVAL, LOG_FETCH_UNIT); - - } private boolean isConfigured() { @@ -162,9 +164,9 @@ private void doInitialSetup() { LOG.info("We will perform the initial setup, grab some coffee and cross your fingers ;-)..."); BackendConfig.INSTANCE.updateSetupStatus(true); - LOG.info("Auto-setup will start in 10 seconds to make sure extensions services are running..."); + LOG.info("Auto-setup will start in 5 seconds to make sure all services are running..."); try { - TimeUnit.SECONDS.sleep(10); + TimeUnit.SECONDS.sleep(5); LOG.info("Starting installation procedure"); new AutoInstallation().startAutoInstallation(); BackendConfig.INSTANCE.updateSetupStatus(false); @@ -205,8 +207,6 @@ public void onExit() { } }); - deregisterService(serviceId()); - LOG.info("Thanks for using Apache StreamPipes - see you next time!"); } @@ -288,16 +288,6 @@ private IPipelineStorage getPipelineStorage() { .getPipelineStorageAPI(); } - - @Override - protected List getServiceTags() { - return Arrays.asList( - DefaultSpServiceTags.CORE, - DefaultSpServiceTags.CONNECT_MASTER, - DefaultSpServiceTags.STREAMPIPES_CLIENT - ); - } - @Override protected String getHealthCheckPath() { return "/streampipes-backend/api/svchealth/" + AUTO_GENERATED_SERVICE_ID; diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesResourceConfig.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesResourceConfig.java index a27595a94d..49c29bdd1d 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesResourceConfig.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/StreamPipesResourceConfig.java @@ -56,14 +56,16 @@ import org.apache.streampipes.rest.impl.Setup; import org.apache.streampipes.rest.impl.UserResource; import org.apache.streampipes.rest.impl.Version; -import org.apache.streampipes.rest.impl.admin.ConsulConfig; import org.apache.streampipes.rest.impl.admin.DataExportResource; import org.apache.streampipes.rest.impl.admin.DataImportResource; import org.apache.streampipes.rest.impl.admin.EmailConfigurationResource; import org.apache.streampipes.rest.impl.admin.ExtensionsServiceEndpointResource; import org.apache.streampipes.rest.impl.admin.GeneralConfigurationResource; +import org.apache.streampipes.rest.impl.admin.MessagingConfigurationResource; import org.apache.streampipes.rest.impl.admin.PermissionResource; import org.apache.streampipes.rest.impl.admin.PipelineElementImport; +import org.apache.streampipes.rest.impl.admin.ServiceConfigurationResource; +import org.apache.streampipes.rest.impl.admin.ServiceRegistrationResource; import org.apache.streampipes.rest.impl.admin.UserAdminResource; import org.apache.streampipes.rest.impl.admin.UserGroupResource; import org.apache.streampipes.rest.impl.connect.AdapterResource; @@ -109,7 +111,6 @@ public Set> getClassesToRegister() { AssetManagementResource.class, AutoComplete.class, CategoryResource.class, - ConsulConfig.class, ContainerProvidedOptions.class, DashboardWidget.class, Dashboard.class, @@ -129,6 +130,7 @@ public Set> getClassesToRegister() { GenericStorageResource.class, LabelResource.class, MeasurementUnitResource.class, + MessagingConfigurationResource.class, Notification.class, OntologyMeasurementUnit.class, PermissionResource.class, @@ -152,7 +154,9 @@ public Set> getClassesToRegister() { Setup.class, ResetResource.class, RestorePasswordResource.class, + ServiceConfigurationResource.class, ServiceHealthResource.class, + ServiceRegistrationResource.class, UserResource.class, UserAdminResource.class, Version.class, diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java index 6a8e81160e..3336f462de 100644 --- a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/AvailableMigrations.java @@ -24,6 +24,7 @@ import org.apache.streampipes.service.core.migrations.v070.CreateFileAssetTypeMigration; import org.apache.streampipes.service.core.migrations.v090.UpdateUsernameViewMigration; import org.apache.streampipes.service.core.migrations.v093.AdapterMigration; +import org.apache.streampipes.service.core.migrations.v093.ConsulConfigMigration; import java.util.Arrays; import java.util.List; @@ -36,7 +37,8 @@ public List getAvailableMigrations() { new CreateDefaultAssetMigration(), new CreateFileAssetTypeMigration(), new UpdateUsernameViewMigration(), - new AdapterMigration() + new AdapterMigration(), + new ConsulConfigMigration() ); } } diff --git a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/ConsulConfigMigration.java b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/ConsulConfigMigration.java new file mode 100644 index 0000000000..899c9a5caf --- /dev/null +++ b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v093/ConsulConfigMigration.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.service.core.migrations.v093; + +import org.apache.streampipes.config.backend.BackendConfig; +import org.apache.streampipes.model.configuration.SpCoreConfiguration; +import org.apache.streampipes.service.core.migrations.Migration; +import org.apache.streampipes.storage.api.ISpCoreConfigurationStorage; +import org.apache.streampipes.storage.management.StorageDispatcher; + +public class ConsulConfigMigration implements Migration { + + private final ISpCoreConfigurationStorage storage; + + public ConsulConfigMigration() { + this.storage = StorageDispatcher.INSTANCE.getNoSqlStore().getSpCoreConfigurationStorage(); + } + + @Override + public boolean shouldExecute() { + return storage.getAll().size() == 0; + } + + @Override + public void executeMigration() { + var currConf = BackendConfig.INSTANCE; + var newConf = new SpCoreConfiguration(); + + var messagingSettings = currConf.getMessagingSettings(); + newConf.setLocalAuthConfig(currConf.getLocalAuthConfig()); + newConf.setEmailConfig(currConf.getEmailConfig()); + newConf.setGeneralConfig(currConf.getGeneralConfig()); + + messagingSettings.setJmsHost(currConf.getJmsHost()); + messagingSettings.setJmsPort(currConf.getJmsPort()); + messagingSettings.setMqttHost(currConf.getMqttHost()); + messagingSettings.setMqttPort(currConf.getMqttPort()); + messagingSettings.setNatsHost(currConf.getNatsHost()); + messagingSettings.setNatsPort(currConf.getNatsPort()); + messagingSettings.setKafkaHost(currConf.getKafkaHost()); + messagingSettings.setKafkaPort(currConf.getKafkaPort()); + messagingSettings.setPulsarUrl(currConf.getPulsarUrl()); + messagingSettings.setZookeeperHost(currConf.getZookeeperHost()); + messagingSettings.setZookeeperPort(currConf.getZookeeperPort()); + newConf.setAssetDir(currConf.getAssetDir()); + newConf.setFilesDir(currConf.getFilesDir()); + + newConf.setMessagingSettings(messagingSettings); + storage.createElement(newConf); + } + + @Override + public String getDescription() { + return "Migrate Consul configurations to CouchDB"; + } +} diff --git a/streampipes-service-discovery-api/pom.xml b/streampipes-service-discovery-api/pom.xml index f56faa9128..1284f7c074 100644 --- a/streampipes-service-discovery-api/pom.xml +++ b/streampipes-service-discovery-api/pom.xml @@ -27,6 +27,17 @@ streampipes-service-discovery-api + + + org.apache.streampipes + streampipes-model + 0.93.0-SNAPSHOT + + + com.google.code.gson + gson + + diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/ISpServiceDiscovery.java b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/ISpServiceDiscovery.java index 729739229f..4f4350f7d8 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/ISpServiceDiscovery.java +++ b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/ISpServiceDiscovery.java @@ -17,20 +17,10 @@ */ package org.apache.streampipes.svcdiscovery.api; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceRegistrationRequest; - import java.util.List; -import java.util.Map; public interface ISpServiceDiscovery { - /** - * Register service. - * - * @param serviceRegistrationRequest the service registration request - */ - void registerService(SpServiceRegistrationRequest serviceRegistrationRequest); - /** * Get active pipeline element service endpoints * @@ -38,13 +28,6 @@ public interface ISpServiceDiscovery { */ List getActivePipelineElementEndpoints(); - /** - * Get active StreamPipes Connect worker endpoints - * - * @return list of StreamPipes Connect worker endpoints - */ - List getActiveConnectWorkerEndpoints(); - /** * Get service endpoints * @@ -56,19 +39,4 @@ public interface ISpServiceDiscovery { List getServiceEndpoints(String svcGroup, boolean restrictToHealthy, List filterByTags); - - /** - * Get all pipeline element service endpoints - * - * @return list of pipline element service endpoints - */ - Map getExtensionsServiceGroups(); - - /** - * Deregister registered service endpoint in Consul - * - * @param svcId service id of endpoint to be deregistered - */ - void deregisterService(String svcId); - } diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/SpConfig.java b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/SpConfig.java index caab3e6ff6..e6b0f8fcf4 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/SpConfig.java +++ b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/SpConfig.java @@ -18,8 +18,8 @@ package org.apache.streampipes.svcdiscovery.api; -import org.apache.streampipes.svcdiscovery.api.model.ConfigItem; -import org.apache.streampipes.svcdiscovery.api.model.ConfigurationScope; +import org.apache.streampipes.model.extensions.configuration.ConfigItem; +import org.apache.streampipes.model.extensions.configuration.ConfigurationScope; public interface SpConfig { diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/DefaultSpServiceTags.java b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/DefaultSpServiceTags.java index 289211c87a..b2b5091805 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/DefaultSpServiceTags.java +++ b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/DefaultSpServiceTags.java @@ -17,12 +17,13 @@ */ package org.apache.streampipes.svcdiscovery.api.model; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; + public class DefaultSpServiceTags { public static final SpServiceTag CORE = SpServiceTag.create(SpServiceTagPrefix.SYSTEM, "core"); public static final SpServiceTag PE = SpServiceTag.create(SpServiceTagPrefix.SYSTEM, "pe"); - public static final SpServiceTag CONNECT_MASTER = SpServiceTag.create(SpServiceTagPrefix.SYSTEM, "connect-master"); - public static final SpServiceTag CONNECT_WORKER = SpServiceTag.create(SpServiceTagPrefix.SYSTEM, "connect-worker"); - public static final SpServiceTag STREAMPIPES_CLIENT = - SpServiceTag.create(SpServiceTagPrefix.SYSTEM, "streampipes-client"); + public static final SpServiceTag CONNECT_WORKER = SpServiceTag + .create(SpServiceTagPrefix.SYSTEM, "connect-worker"); } diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/DefaultSpServiceGroups.java b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/DefaultSpServiceTypes.java similarity index 91% rename from streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/DefaultSpServiceGroups.java rename to streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/DefaultSpServiceTypes.java index 5fd3452f39..6e343a809b 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/DefaultSpServiceGroups.java +++ b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/DefaultSpServiceTypes.java @@ -17,9 +17,8 @@ */ package org.apache.streampipes.svcdiscovery.api.model; -public class DefaultSpServiceGroups { +public class DefaultSpServiceTypes { - public static final String CORE = "core"; public static final String EXT = "ext"; } diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/PeConfig.java b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/PeConfig.java index d0053e8bf4..9b852168fe 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/PeConfig.java +++ b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/PeConfig.java @@ -18,6 +18,8 @@ package org.apache.streampipes.svcdiscovery.api.model; +import org.apache.streampipes.model.extensions.configuration.ConfigItem; + import java.util.List; import java.util.Map; diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceRegistrationRequest.java b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceRegistrationRequest.java deleted file mode 100644 index 0b0e6c0d28..0000000000 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceRegistrationRequest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.svcdiscovery.api.model; - -import java.util.List; - -public class SpServiceRegistrationRequest { - - private String svcGroup; - private String svcId; - private String host; - private int port; - private List tags; - private String healthCheckPath; - - public SpServiceRegistrationRequest(String svcGroup, - String svcId, - String host, - int port, - List tags, - String healthCheckPath) { - this.svcGroup = svcGroup; - this.svcId = svcId; - this.host = host; - this.port = port; - this.tags = tags; - this.healthCheckPath = healthCheckPath; - } - - public static SpServiceRegistrationRequest from(String svcGroup, - String svcId, - String host, - Integer port, - List tags) { - return new SpServiceRegistrationRequest(svcGroup, svcId, host, port, tags, ""); - } - - public static SpServiceRegistrationRequest from(String svcGroup, - String svcId, - String host, - Integer port, - List tags, - String healthCheckPath) { - return new SpServiceRegistrationRequest(svcGroup, svcId, host, port, tags, healthCheckPath); - } - - public String getSvcGroup() { - return svcGroup; - } - - public void setSvcGroup(String svcGroup) { - this.svcGroup = svcGroup; - } - - public String getSvcId() { - return svcId; - } - - public void setSvcId(String svcId) { - this.svcId = svcId; - } - - public String getHost() { - return host; - } - - public void setHost(String host) { - this.host = host; - } - - public int getPort() { - return port; - } - - public void setPort(int port) { - this.port = port; - } - - public List getTags() { - return tags; - } - - public void setTags(List tags) { - this.tags = tags; - } - - public String getHealthCheckPath() { - return healthCheckPath; - } - - public void setHealthCheckPath(String healthCheckPath) { - this.healthCheckPath = healthCheckPath; - } -} diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceUrlProvider.java b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceUrlProvider.java index 1e02adf148..0e0da154e8 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceUrlProvider.java +++ b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/model/SpServiceUrlProvider.java @@ -17,6 +17,9 @@ */ package org.apache.streampipes.svcdiscovery.api.model; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; + public enum SpServiceUrlProvider { DATA_PROCESSOR(SpServicePathPrefix.DATA_PROCESSOR, SpServiceTagPrefix.DATA_PROCESSOR), diff --git a/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulHealthServiceManager.java b/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulHealthServiceManager.java deleted file mode 100644 index cb1710483e..0000000000 --- a/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulHealthServiceManager.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.svcdiscovery.consul; - -import org.apache.streampipes.commons.environment.Environment; - -import com.ecwid.consul.v1.QueryParams; -import com.ecwid.consul.v1.Response; -import com.ecwid.consul.v1.health.HealthServicesRequest; -import com.ecwid.consul.v1.health.model.Check; -import com.ecwid.consul.v1.health.model.HealthService; - -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -public class ConsulHealthServiceManager extends AbstractConsulService { - - private static final int MAX_RETRIES = 3; - - public ConsulHealthServiceManager(Environment environment) { - super(environment); - initializeCache(); - } - - public void initializeCache() { - - } - - public List getServiceEndpoints(String serviceGroup, - boolean restrictToHealthy, - List filterByTags) { - List activeServices = findService(serviceGroup, 0); - - return activeServices - .stream() - .filter(service -> allFiltersSupported(service, filterByTags)) - .filter(service -> !restrictToHealthy - || service.getChecks().stream().allMatch(check -> check.getStatus() == Check.CheckStatus.PASSING)) - .map(this::makeServiceUrl) - .collect(Collectors.toList()); - } - - private String makeServiceUrl(HealthService service) { - return service.getService().getAddress() + ":" + service.getService().getPort(); - } - - private boolean allFiltersSupported(HealthService service, - List filterByTags) { - return new HashSet<>(service.getService().getTags()).containsAll(filterByTags); - } - - private List findService(String serviceGroup, int retryCount) { - HealthServicesRequest request = HealthServicesRequest.newBuilder() - .setPassing(true) - .setQueryParams(QueryParams.DEFAULT) - .build(); - Response> healthyServicesResp = consulInstance().getHealthServices(serviceGroup, request); - var healthyServices = healthyServicesResp.getValue(); - if (healthyServices.size() == 0) { - if (retryCount < MAX_RETRIES) { - try { - retryCount++; - TimeUnit.SECONDS.sleep(1); - return findService(serviceGroup, retryCount); - } catch (InterruptedException e) { - e.printStackTrace(); - return Collections.emptyList(); - } - } else { - return Collections.emptyList(); - } - } else { - return healthyServices; - } - } - - -} diff --git a/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulSpConfig.java b/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulSpConfig.java index 73770eeead..ad67689c8f 100644 --- a/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulSpConfig.java +++ b/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulSpConfig.java @@ -19,11 +19,11 @@ package org.apache.streampipes.svcdiscovery.consul; import org.apache.streampipes.commons.environment.Environment; +import org.apache.streampipes.model.extensions.configuration.ConfigItem; +import org.apache.streampipes.model.extensions.configuration.ConfigItemUtils; +import org.apache.streampipes.model.extensions.configuration.ConfigurationScope; import org.apache.streampipes.serializers.json.JacksonSerializer; import org.apache.streampipes.svcdiscovery.api.SpConfig; -import org.apache.streampipes.svcdiscovery.api.model.ConfigItem; -import org.apache.streampipes.svcdiscovery.api.model.ConfigItemUtils; -import org.apache.streampipes.svcdiscovery.api.model.ConfigurationScope; import com.ecwid.consul.v1.ConsulClient; import com.fasterxml.jackson.core.JsonProcessingException; diff --git a/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/SpConsulKvManagement.java b/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/SpConsulKvManagement.java deleted file mode 100644 index 31247feca6..0000000000 --- a/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/SpConsulKvManagement.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.svcdiscovery.consul; - -import org.apache.streampipes.commons.environment.Environment; -import org.apache.streampipes.svcdiscovery.api.ISpKvManagement; - -import com.ecwid.consul.v1.Response; -import com.ecwid.consul.v1.kv.model.GetValue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class SpConsulKvManagement extends AbstractConsulService implements ISpKvManagement { - - private static final Logger LOG = LoggerFactory.getLogger(SpConsulKvManagement.class); - - private static final String CONSUL_NAMESPACE = "/sp/v1/"; - - public SpConsulKvManagement(Environment environment) { - super(environment); - } - - public Map getKeyValue(String route) { - var consul = consulInstance(); - - Map keyValues = new HashMap<>(); - Response> consulResponseWithValues = consul.getKVValues(route); - - if (consulResponseWithValues.getValue() != null) { - for (GetValue value : consulResponseWithValues.getValue()) { - String key = value.getKey(); - String v = ""; - if (value.getValue() != null) { - v = value.getDecodedValue(); - } - keyValues.put(key, v); - } - } - return keyValues; - } - - public void updateConfig(String key, String entry, boolean password) { - var consul = consulInstance(); - if (!password) { - LOG.info("Updated config - key:" + key + " value: " + entry); - consul.setKVValue(key, entry); - } - } - - public void deleteConfig(String key) { - var consul = consulInstance(); - LOG.info("Delete config: {}", key); - consul.deleteKVValue(CONSUL_NAMESPACE + key); - } -} diff --git a/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/SpConsulServiceDiscovery.java b/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/SpConsulServiceDiscovery.java deleted file mode 100644 index a00acf5752..0000000000 --- a/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/SpConsulServiceDiscovery.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.svcdiscovery.consul; - -import org.apache.streampipes.commons.environment.Environment; -import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceGroups; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTags; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceRegistrationRequest; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceTag; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceTagPrefix; - -import com.ecwid.consul.v1.Response; -import com.ecwid.consul.v1.agent.model.Check; -import com.ecwid.consul.v1.agent.model.NewService; -import com.ecwid.consul.v1.agent.model.Service; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - - -public class SpConsulServiceDiscovery extends AbstractConsulService implements ISpServiceDiscovery { - - private static final Logger LOG = LoggerFactory.getLogger(SpConsulServiceDiscovery.class); - - private static final String HTTP_PROTOCOL = "http://"; - private static final String COLON = ":"; - private static final String HEALTH_CHECK_INTERVAL = "10s"; - - private ConsulHealthServiceManager healthServiceManager; - - public SpConsulServiceDiscovery(Environment environment) { - super(environment); - this.healthServiceManager = new ConsulHealthServiceManager(environment); - } - - @Override - public void registerService(SpServiceRegistrationRequest req) { - consulInstance().agentServiceRegister(createRegistrationBody(req)); - LOG.info("Successfully registered service at Consul: " + req.getSvcId()); - } - - private List asString(List tags) { - return tags.stream().map(SpServiceTag::asString).collect(Collectors.toList()); - } - - @Override - public List getActivePipelineElementEndpoints() { - LOG.info("Discovering active pipeline element service endpoints"); - return getServiceEndpoints(DefaultSpServiceGroups.EXT, true, - Collections.singletonList(DefaultSpServiceTags.PE.asString())); - } - - @Override - public List getActiveConnectWorkerEndpoints() { - LOG.info("Discovering active StreamPipes Connect worker service endpoints"); - return getServiceEndpoints(DefaultSpServiceGroups.EXT, true, - Collections.singletonList(DefaultSpServiceTags.CONNECT_WORKER.asString())); - } - - @Override - public List getServiceEndpoints(String svcGroup, boolean restrictToHealthy, List filterByTags) { - return healthServiceManager.getServiceEndpoints(svcGroup, restrictToHealthy, filterByTags); - } - - @Override - public Map getExtensionsServiceGroups() { - LOG.info("Load pipeline element service status"); - var consul = consulInstance(); - - Response> servicesResp = consul.getAgentServices(); - Response> checksResp = consul.getAgentChecks(); - - Map peSvcs = new HashMap<>(); - if (servicesResp.getValue() != null) { - var services = servicesResp.getValue(); - var checks = checksResp.getValue(); - for (Map.Entry entry : services.entrySet()) { - if (hasExtensionsTag(entry.getValue().getTags())) { - String serviceId = entry.getValue().getId(); - String serviceStatus = "critical"; - if (checks.containsKey("service:" + entry.getKey())) { - serviceStatus = checks.get("service:" + entry.getKey()).getStatus().name(); - } - LOG.info("Service id: " + serviceId + " service status: " + serviceStatus); - String serviceGroup = extractServiceGroup(entry.getValue().getTags()); - peSvcs.put(serviceGroup, serviceStatus); - } - } - } - return peSvcs; - } - - private boolean hasExtensionsTag(List tags) { - return tags.stream().anyMatch(tag -> tag.equals(DefaultSpServiceTags.PE.asString()) - || tag.equals(DefaultSpServiceTags.CONNECT_WORKER.asString())); - } - - private String extractServiceGroup(List tags) { - String groupTag = tags.stream().filter(tag -> tag.startsWith(SpServiceTagPrefix.SP_GROUP.asString())).findFirst() - .orElse("unknown service group"); - return groupTag.replaceAll(SpServiceTagPrefix.SP_GROUP.asString() + ":", ""); - } - - @Override - public void deregisterService(String svcId) { - var consul = consulInstance(); - LOG.info("Deregister service: " + svcId); - consul.agentServiceDeregister(svcId); - } - - private NewService createRegistrationBody(SpServiceRegistrationRequest req) { - var service = new NewService(); - service.setId(req.getSvcId()); - service.setName(req.getSvcGroup()); - service.setPort(req.getPort()); - service.setAddress(HTTP_PROTOCOL + req.getHost()); - service.setCheck(createServiceCheck(req)); - - service.setTags(asString(req.getTags())); - service.setEnableTagOverride(true); - - return service; - } - - private NewService.Check createServiceCheck(SpServiceRegistrationRequest req) { - var serviceCheck = new NewService.Check(); - - serviceCheck.setHttp(HTTP_PROTOCOL + req.getHost() + COLON + req.getPort() + req.getHealthCheckPath()); - serviceCheck.setInterval(HEALTH_CHECK_INTERVAL); - serviceCheck.setDeregisterCriticalServiceAfter("120s"); - serviceCheck.setStatus("passing"); - - return serviceCheck; - } -} diff --git a/streampipes-service-discovery/pom.xml b/streampipes-service-discovery/pom.xml index c32772c1a0..f7bb3c11f5 100644 --- a/streampipes-service-discovery/pom.xml +++ b/streampipes-service-discovery/pom.xml @@ -28,6 +28,11 @@ streampipes-service-discovery + + org.apache.streampipes + streampipes-commons + 0.93.0-SNAPSHOT + org.apache.streampipes streampipes-service-discovery-api @@ -38,6 +43,11 @@ streampipes-service-discovery-consul 0.93.0-SNAPSHOT + + org.apache.streampipes + streampipes-storage-management + 0.93.0-SNAPSHOT + diff --git a/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscovery.java b/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscovery.java index cead1334b0..e46169a043 100644 --- a/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscovery.java +++ b/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscovery.java @@ -19,29 +19,18 @@ import org.apache.streampipes.commons.environment.Environment; import org.apache.streampipes.commons.environment.Environments; -import org.apache.streampipes.svcdiscovery.api.ISpKvManagement; import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery; import org.apache.streampipes.svcdiscovery.api.SpConfig; import org.apache.streampipes.svcdiscovery.consul.ConsulSpConfig; -import org.apache.streampipes.svcdiscovery.consul.SpConsulKvManagement; -import org.apache.streampipes.svcdiscovery.consul.SpConsulServiceDiscovery; public class SpServiceDiscovery { public static ISpServiceDiscovery getServiceDiscovery() { - return new SpConsulServiceDiscovery(Environments.getEnvironment()); + return new SpServiceDiscoveryCore(); } public static ISpServiceDiscovery getServiceDiscovery(Environment environment) { - return new SpConsulServiceDiscovery(environment); - } - - public static ISpKvManagement getKeyValueStore() { - return new SpConsulKvManagement(Environments.getEnvironment()); - } - - public static ISpKvManagement getKeyValueStore(Environment environment) { - return new SpConsulKvManagement(environment); + return new SpServiceDiscoveryCore(); } public static SpConfig getSpConfig(String serviceGroup) { diff --git a/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscoveryCore.java b/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscoveryCore.java new file mode 100644 index 0000000000..2ad6bed1f8 --- /dev/null +++ b/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscoveryCore.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.svcdiscovery; + +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.storage.api.CRUDStorage; +import org.apache.streampipes.storage.management.StorageDispatcher; +import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery; +import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTags; +import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTypes; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class SpServiceDiscoveryCore implements ISpServiceDiscovery { + + private static final Logger LOG = LoggerFactory.getLogger(SpServiceDiscoveryCore.class); + private static final int MAX_RETRIES = 3; + + private final CRUDStorage serviceStorage; + + public SpServiceDiscoveryCore() { + this.serviceStorage = StorageDispatcher.INSTANCE.getNoSqlStore().getExtensionsServiceStorage(); + } + + @Override + public List getActivePipelineElementEndpoints() { + LOG.info("Discovering active pipeline element service endpoints"); + return getServiceEndpoints(DefaultSpServiceTypes.EXT, true, + Collections.singletonList(DefaultSpServiceTags.PE.asString())); + } + + @Override + public List getServiceEndpoints(String serviceGroup, + boolean restrictToHealthy, + List filterByTags) { + List activeServices = findService(0); + + return activeServices + .stream() + .filter(service -> allFiltersSupported(service, filterByTags)) + .filter(service -> !restrictToHealthy + || service.isHealthy()) + .map(this::makeServiceUrl) + .collect(Collectors.toList()); + } + + private String makeServiceUrl(SpServiceRegistration service) { + return service.getScheme() + "://" + service.getHost() + ":" + service.getPort(); + } + + private boolean allFiltersSupported(SpServiceRegistration service, + List filterByTags) { + return new HashSet<>(service.getTags()) + .stream() + .anyMatch(tag -> filterByTags.contains(tag.asString())); + } + + private List findService(int retryCount) { + var services = serviceStorage.getAll(); + if (services.size() == 0) { + if (retryCount < MAX_RETRIES) { + try { + retryCount++; + TimeUnit.SECONDS.sleep(10); + return findService(retryCount); + } catch (InterruptedException e) { + e.printStackTrace(); + return Collections.emptyList(); + } + } else { + return Collections.emptyList(); + } + } else { + return services; + } + } +} diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java index 6f0291c0c6..a67e9175eb 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ExtensionsModelSubmitter.java @@ -19,13 +19,11 @@ import org.apache.streampipes.extensions.management.init.DeclarersSingleton; import org.apache.streampipes.extensions.management.model.SpServiceDefinition; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; import org.apache.streampipes.service.extensions.connect.ConnectWorkerRegistrationService; import org.apache.streampipes.service.extensions.function.StreamPipesFunctionHandler; import org.apache.streampipes.service.extensions.security.WebSecurityConfig; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceTag; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; @@ -38,8 +36,6 @@ @EnableAutoConfiguration @Import({ExtensionsResourceConfig.class, WebSecurityConfig.class}) public abstract class ExtensionsModelSubmitter extends StreamPipesExtensionsServiceBase { - private static final Logger LOG = - LoggerFactory.getLogger(ExtensionsModelSubmitter.class.getCanonicalName()); @PreDestroy public void onExit() { diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ServiceTagProvider.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ServiceTagProvider.java index 30e13b8a93..f565ec46e7 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ServiceTagProvider.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/ServiceTagProvider.java @@ -22,8 +22,8 @@ import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement; import org.apache.streampipes.extensions.management.init.DeclarersSingleton; import org.apache.streampipes.extensions.management.util.ServiceDefinitionUtil; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTags; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceTag; import java.util.ArrayList; import java.util.Collection; diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java index b634989164..c13b1153a8 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java @@ -18,13 +18,19 @@ package org.apache.streampipes.service.extensions; +import org.apache.streampipes.client.StreamPipesClient; +import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver; import org.apache.streampipes.extensions.management.init.DeclarersSingleton; import org.apache.streampipes.extensions.management.model.SpServiceDefinition; +import org.apache.streampipes.model.extensions.configuration.ConfigItem; +import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.service.base.BaseNetworkingConfig; import org.apache.streampipes.service.base.StreamPipesServiceBase; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceGroups; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceTag; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceTagPrefix; +import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTypes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,10 +40,12 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; public abstract class StreamPipesExtensionsServiceBase extends StreamPipesServiceBase { private static final Logger LOG = LoggerFactory.getLogger(StreamPipesExtensionsServiceBase.class); + private static final int RETRY_INTERVAL_SECONDS = 3; public void init() { SpServiceDefinition serviceDef = provideServiceDefinition(); @@ -59,25 +67,54 @@ public void init(SpServiceDefinition serviceDef) { } } - public SpServiceDefinition provideServiceDefinition() { - return null; - } + public abstract SpServiceDefinition provideServiceDefinition(); public abstract void afterServiceRegistered(SpServiceDefinition serviceDef); public void startExtensionsService(Class serviceClass, SpServiceDefinition serviceDef, BaseNetworkingConfig networkingConfig) throws UnknownHostException { + var req = SpServiceRegistration.from( + DefaultSpServiceTypes.EXT, + serviceDef.getServiceGroup(), + serviceId(), + networkingConfig.getHost(), + networkingConfig.getPort(), + getServiceTags(), + getHealthCheckPath()); + + LOG.info("Registering service {} with id {} at core", req.getSvcGroup(), req.getSvcId()); + registerService(req); + + this.registerConfigs(serviceDef.getServiceGroup(), serviceDef.getServiceName(), serviceDef.getKvConfigs()); this.startStreamPipesService( serviceClass, - DefaultSpServiceGroups.EXT, - serviceId(), networkingConfig ); + this.afterServiceRegistered(serviceDef); } - @Override + private void registerService(SpServiceRegistration serviceRegistration) { + StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); + try { + client.adminApi().registerService(serviceRegistration); + LOG.info("Successfully registered service at core."); + } catch (SpRuntimeException e) { + LOG.warn( + "Could not register at core at url {}. Trying again in {} seconds", + client.getConnectionConfig().getBaseUrl(), + RETRY_INTERVAL_SECONDS + ); + try { + TimeUnit.SECONDS.sleep(RETRY_INTERVAL_SECONDS); + registerService(serviceRegistration); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + } + } + protected List getServiceTags() { List tags = new ArrayList<>(); if (DeclarersSingleton.getInstance().getServiceDefinition() != null) { @@ -88,6 +125,21 @@ protected List getServiceTags() { return tags; } + protected void deregisterService(String serviceId) { + LOG.info("Deregistering service (id={})...", serviceId); + StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); + client.adminApi().deregisterService(serviceId); + } + + private void registerConfigs(String serviceGroup, + String serviceName, + List configs) { + LOG.info("Registering {} service configs for service {}", configs.size(), serviceGroup); + StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); + var serviceConfiguration = new SpServiceConfiguration(serviceGroup, serviceName, configs); + client.adminApi().registerServiceConfiguration(serviceConfiguration); + } + protected abstract List getExtensionsServiceTags(); @PreDestroy diff --git a/streampipes-sources/src/main/java/org/apache/streampipes/sources/AbstractAdapterIncludedStream.java b/streampipes-sources/src/main/java/org/apache/streampipes/sources/AbstractAdapterIncludedStream.java index 7f31edcda8..5c1cba4a28 100644 --- a/streampipes-sources/src/main/java/org/apache/streampipes/sources/AbstractAdapterIncludedStream.java +++ b/streampipes-sources/src/main/java/org/apache/streampipes/sources/AbstractAdapterIncludedStream.java @@ -33,7 +33,7 @@ public boolean isExecutable() { } public ConfigExtractor configExtractor() { - return ConfigExtractor.from(DeclarersSingleton.getInstance().getServiceDefinition().getServiceGroup()); + return ConfigExtractor.from(DeclarersSingleton.getInstance().getServiceGroup()); } @Override diff --git a/streampipes-sources/src/main/java/org/apache/streampipes/sources/AbstractAlreadyExistingStream.java b/streampipes-sources/src/main/java/org/apache/streampipes/sources/AbstractAlreadyExistingStream.java index 42eb4705be..d08d435b06 100644 --- a/streampipes-sources/src/main/java/org/apache/streampipes/sources/AbstractAlreadyExistingStream.java +++ b/streampipes-sources/src/main/java/org/apache/streampipes/sources/AbstractAlreadyExistingStream.java @@ -39,7 +39,7 @@ public void executeStream() { } public ConfigExtractor configExtractor() { - return ConfigExtractor.from(DeclarersSingleton.getInstance().getServiceDefinition().getServiceGroup()); + return ConfigExtractor.from(DeclarersSingleton.getInstance().getServiceGroup()); } @Override diff --git a/streampipes-storage-api/pom.xml b/streampipes-storage-api/pom.xml index 754c9704bf..15ae3f044a 100644 --- a/streampipes-storage-api/pom.xml +++ b/streampipes-storage-api/pom.xml @@ -39,6 +39,11 @@ streampipes-model-client 0.93.0-SNAPSHOT + + org.apache.streampipes + streampipes-service-discovery-api + 0.93.0-SNAPSHOT + diff --git a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/INoSqlStorage.java b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/INoSqlStorage.java index 6b4af153a7..01e9fdbea6 100644 --- a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/INoSqlStorage.java +++ b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/INoSqlStorage.java @@ -17,6 +17,9 @@ */ package org.apache.streampipes.storage.api; +import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; + public interface INoSqlStorage { IGenericStorage getGenericStorage(); @@ -80,4 +83,10 @@ public interface INoSqlStorage { IPasswordRecoveryTokenStorage getPasswordRecoveryTokenStorage(); IUserActivationTokenStorage getUserActivationTokenStorage(); + + CRUDStorage getExtensionsServiceStorage(); + + CRUDStorage getExtensionsServiceConfigurationStorage(); + + ISpCoreConfigurationStorage getSpCoreConfigurationStorage(); } diff --git a/ui/src/app/configuration/shared/streampipes-pe-container.model.ts b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/ISpCoreConfigurationStorage.java similarity index 66% rename from ui/src/app/configuration/shared/streampipes-pe-container.model.ts rename to streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/ISpCoreConfigurationStorage.java index 5061bad88b..829dc09ce9 100644 --- a/ui/src/app/configuration/shared/streampipes-pe-container.model.ts +++ b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/ISpCoreConfigurationStorage.java @@ -1,29 +1,36 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import { StreampipesPeContainerConifgs } from './streampipes-pe-container-configs'; - -// ConsulService = StreampipesPeContainer ERLEDIGT -export interface StreampipesPeContainer { - name: string; - mainKey: string; - meta: { - status: string; - }; - configs: [StreampipesPeContainerConifgs]; -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.storage.api; + +import org.apache.streampipes.model.configuration.SpCoreConfiguration; + +import java.util.List; + +public interface ISpCoreConfigurationStorage { + + List getAll(); + + void createElement(SpCoreConfiguration element); + + SpCoreConfiguration get(); + + SpCoreConfiguration updateElement(SpCoreConfiguration element); + + void deleteElement(); +} diff --git a/streampipes-storage-couchdb/pom.xml b/streampipes-storage-couchdb/pom.xml index 963d0a25cc..30d1efb7b2 100644 --- a/streampipes-storage-couchdb/pom.xml +++ b/streampipes-storage-couchdb/pom.xml @@ -31,7 +31,7 @@ org.apache.streampipes - streampipes-service-discovery + streampipes-service-discovery-api 0.93.0-SNAPSHOT diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java index fb9c28ca69..988b45ff80 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/CouchDbStorageManager.java @@ -17,6 +17,9 @@ */ package org.apache.streampipes.storage.couchdb; +import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.storage.api.CRUDStorage; import org.apache.streampipes.storage.api.IAdapterStorage; import org.apache.streampipes.storage.api.IAssetDashboardStorage; import org.apache.streampipes.storage.api.ICategoryStorage; @@ -43,6 +46,7 @@ import org.apache.streampipes.storage.api.IPipelineElementTemplateStorage; import org.apache.streampipes.storage.api.IPipelineMonitoringDataStorage; import org.apache.streampipes.storage.api.IPipelineStorage; +import org.apache.streampipes.storage.api.ISpCoreConfigurationStorage; import org.apache.streampipes.storage.api.IUserActivationTokenStorage; import org.apache.streampipes.storage.api.IUserGroupStorage; import org.apache.streampipes.storage.api.IUserStorage; @@ -52,6 +56,7 @@ import org.apache.streampipes.storage.couchdb.impl.AssetDashboardStorageImpl; import org.apache.streampipes.storage.couchdb.impl.CategoryStorageImpl; import org.apache.streampipes.storage.couchdb.impl.ConnectionStorageImpl; +import org.apache.streampipes.storage.couchdb.impl.CoreConfigurationStorageImpl; import org.apache.streampipes.storage.couchdb.impl.DashboardStorageImpl; import org.apache.streampipes.storage.couchdb.impl.DashboardWidgetStorageImpl; import org.apache.streampipes.storage.couchdb.impl.DataExplorerDashboardStorageImpl; @@ -60,7 +65,9 @@ import org.apache.streampipes.storage.couchdb.impl.DataProcessorStorageImpl; import org.apache.streampipes.storage.couchdb.impl.DataSinkStorageImpl; import org.apache.streampipes.storage.couchdb.impl.DataStreamStorageImpl; +import org.apache.streampipes.storage.couchdb.impl.ExtensionsServiceConfigStorageImpl; import org.apache.streampipes.storage.couchdb.impl.ExtensionsServiceEndpointStorageImpl; +import org.apache.streampipes.storage.couchdb.impl.ExtensionsServiceStorageImpl; import org.apache.streampipes.storage.couchdb.impl.FileMetadataStorageImpl; import org.apache.streampipes.storage.couchdb.impl.GenericStorageImpl; import org.apache.streampipes.storage.couchdb.impl.ImageStorageImpl; @@ -239,5 +246,20 @@ public IUserActivationTokenStorage getUserActivationTokenStorage() { return new UserActivationTokenImpl(); } + @Override + public CRUDStorage getExtensionsServiceStorage() { + return new ExtensionsServiceStorageImpl(); + } + + @Override + public CRUDStorage getExtensionsServiceConfigurationStorage() { + return new ExtensionsServiceConfigStorageImpl(); + } + + @Override + public ISpCoreConfigurationStorage getSpCoreConfigurationStorage() { + return new CoreConfigurationStorageImpl(); + } + } diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/CoreConfigurationStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/CoreConfigurationStorageImpl.java new file mode 100644 index 0000000000..7ff0a99c04 --- /dev/null +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/CoreConfigurationStorageImpl.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.storage.couchdb.impl; + +import org.apache.streampipes.model.configuration.SpCoreConfiguration; +import org.apache.streampipes.storage.api.ISpCoreConfigurationStorage; +import org.apache.streampipes.storage.couchdb.dao.AbstractDao; +import org.apache.streampipes.storage.couchdb.utils.Utils; + +import java.util.List; + +public class CoreConfigurationStorageImpl extends AbstractDao + implements ISpCoreConfigurationStorage { + + + public CoreConfigurationStorageImpl() { + super(Utils::getCouchDbGeneralConfigStorage, SpCoreConfiguration.class); + } + + @Override + public List getAll() { + return findAll(); + } + + @Override + public void createElement(SpCoreConfiguration element) { + persist(element); + } + + @Override + public SpCoreConfiguration get() { + return findWithNullIfEmpty(SpCoreConfiguration.ID); + } + + @Override + public SpCoreConfiguration updateElement(SpCoreConfiguration element) { + update(element); + return get(); + } + + @Override + public void deleteElement() { + delete(SpCoreConfiguration.ID); + } +} diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/ExtensionsServiceConfigStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/ExtensionsServiceConfigStorageImpl.java new file mode 100644 index 0000000000..72bee5d5b5 --- /dev/null +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/ExtensionsServiceConfigStorageImpl.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.storage.couchdb.impl; + +import org.apache.streampipes.model.extensions.configuration.SpServiceConfiguration; +import org.apache.streampipes.storage.api.CRUDStorage; +import org.apache.streampipes.storage.couchdb.dao.AbstractDao; +import org.apache.streampipes.storage.couchdb.utils.Utils; + +import java.util.List; + +public class ExtensionsServiceConfigStorageImpl extends AbstractDao + implements CRUDStorage { + + public ExtensionsServiceConfigStorageImpl() { + super(Utils::getCouchDbServiceConfigStorage, SpServiceConfiguration.class); + } + + @Override + public List getAll() { + return findAll(); + } + + @Override + public void createElement(SpServiceConfiguration element) { + persist(element); + } + + @Override + public SpServiceConfiguration getElementById(String id) { + return findWithNullIfEmpty(id); + } + + @Override + public SpServiceConfiguration updateElement(SpServiceConfiguration element) { + update(element); + return getElementById(element.getServiceName()); + } + + @Override + public void deleteElement(SpServiceConfiguration element) { + delete(element.getServiceName()); + } +} diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/ExtensionsServiceStorageImpl.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/ExtensionsServiceStorageImpl.java new file mode 100644 index 0000000000..85a36e85d7 --- /dev/null +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/impl/ExtensionsServiceStorageImpl.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.storage.couchdb.impl; + +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; +import org.apache.streampipes.storage.api.CRUDStorage; +import org.apache.streampipes.storage.couchdb.dao.AbstractDao; +import org.apache.streampipes.storage.couchdb.utils.Utils; + +import java.util.List; + +public class ExtensionsServiceStorageImpl extends AbstractDao + implements CRUDStorage { + + public ExtensionsServiceStorageImpl() { + super(Utils::getCouchDbExtensionsStorage, SpServiceRegistration.class); + } + + + @Override + public List getAll() { + return findAll(); + } + + @Override + public void createElement(SpServiceRegistration element) { + persist(element); + } + + @Override + public SpServiceRegistration getElementById(String id) { + return find(id).orElseThrow(IllegalArgumentException::new); + } + + @Override + public SpServiceRegistration updateElement(SpServiceRegistration element) { + update(element); + return getElementById(element.getSvcId()); + } + + @Override + public void deleteElement(SpServiceRegistration element) { + delete(element.getSvcId()); + } +} diff --git a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/utils/Utils.java b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/utils/Utils.java index 125270494a..b9b48c3036 100644 --- a/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/utils/Utils.java +++ b/streampipes-storage-couchdb/src/main/java/org/apache/streampipes/storage/couchdb/utils/Utils.java @@ -60,10 +60,22 @@ public static CouchDbClient getCouchDbPipelineCanvasMetadataClient() { return getCouchDbGsonClient("pipelinecanvasmetadata"); } + public static CouchDbClient getCouchDbGeneralConfigStorage() { + return getCouchDbGsonClient("general-configuration"); + } + public static CouchDbClient getCouchDbCategoryClient() { return getCouchDbGsonClient("category"); } + public static CouchDbClient getCouchDbServiceConfigStorage() { + return getCouchDbGsonClient("extensions-services-configurations"); + } + + public static CouchDbClient getCouchDbExtensionsStorage() { + return getCouchDbGsonClient("extensions-services"); + } + public static CouchDbClient getCouchDbLabelClient() { return getCouchDbGsonClient("label"); } diff --git a/streampipes-user-management/src/main/java/org/apache/streampipes/user/management/jwt/JwtTokenProvider.java b/streampipes-user-management/src/main/java/org/apache/streampipes/user/management/jwt/JwtTokenProvider.java index 4abb09a3c0..c4af4619eb 100644 --- a/streampipes-user-management/src/main/java/org/apache/streampipes/user/management/jwt/JwtTokenProvider.java +++ b/streampipes-user-management/src/main/java/org/apache/streampipes/user/management/jwt/JwtTokenProvider.java @@ -20,13 +20,14 @@ import org.apache.streampipes.commons.environment.Environment; import org.apache.streampipes.commons.environment.Environments; -import org.apache.streampipes.config.backend.BackendConfig; -import org.apache.streampipes.config.backend.model.JwtSigningMode; -import org.apache.streampipes.config.backend.model.LocalAuthConfig; import org.apache.streampipes.model.client.user.Principal; +import org.apache.streampipes.model.configuration.JwtSigningMode; +import org.apache.streampipes.model.configuration.LocalAuthConfig; +import org.apache.streampipes.model.configuration.SpCoreConfiguration; import org.apache.streampipes.security.jwt.JwtTokenGenerator; import org.apache.streampipes.security.jwt.JwtTokenUtils; import org.apache.streampipes.security.jwt.JwtTokenValidator; +import org.apache.streampipes.storage.management.StorageDispatcher; import org.apache.streampipes.user.management.model.PrincipalUserDetails; import org.apache.streampipes.user.management.util.GrantedAuthoritiesBuilder; import org.apache.streampipes.user.management.util.UserInfoUtil; @@ -51,11 +52,16 @@ public class JwtTokenProvider { public static final String CLAIM_USER = "user"; private static final Logger LOG = LoggerFactory.getLogger(JwtTokenProvider.class); - private BackendConfig config; + private SpCoreConfiguration config; private Environment env; public JwtTokenProvider() { - this.config = BackendConfig.INSTANCE; + this.config = StorageDispatcher + .INSTANCE + .getNoSqlStore() + .getSpCoreConfigurationStorage() + .get(); + this.env = Environments.getEnvironment(); } diff --git a/streampipes-user-management/src/main/java/org/apache/streampipes/user/management/jwt/SpKeyResolver.java b/streampipes-user-management/src/main/java/org/apache/streampipes/user/management/jwt/SpKeyResolver.java index 7e0af50408..6086713835 100644 --- a/streampipes-user-management/src/main/java/org/apache/streampipes/user/management/jwt/SpKeyResolver.java +++ b/streampipes-user-management/src/main/java/org/apache/streampipes/user/management/jwt/SpKeyResolver.java @@ -17,7 +17,6 @@ */ package org.apache.streampipes.user.management.jwt; -import org.apache.streampipes.config.backend.BackendConfig; import org.apache.streampipes.model.client.user.Principal; import org.apache.streampipes.model.client.user.ServiceAccount; import org.apache.streampipes.model.client.user.UserAccount; @@ -70,7 +69,13 @@ private boolean isRealUser(Principal principal) { } public String getPublicKeyFromConfig() { - return BackendConfig.INSTANCE.getLocalAuthConfig().getPublicKey(); + return StorageDispatcher + .INSTANCE + .getNoSqlStore() + .getSpCoreConfigurationStorage() + .get() + .getLocalAuthConfig() + .getPublicKey(); } diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts index ca4deba8b4..36a810b513 100644 --- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts +++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts @@ -20,7 +20,7 @@ /* tslint:disable */ /* eslint-disable */ // @ts-nocheck -// Generated using typescript-generator version 3.1.1185 on 2023-07-23 10:14:46. +// Generated using typescript-generator version 3.1.1185 on 2023-08-06 11:37:37. export class NamedStreamPipesEntity { '@class': @@ -763,6 +763,29 @@ export class ColorPickerStaticProperty extends StaticProperty { } } +export class ConfigItem { + configurationScope: ConfigurationScope; + description: string; + key: string; + password: boolean; + value: string; + valueType: string; + + static fromData(data: ConfigItem, target?: ConfigItem): ConfigItem { + if (!data) { + return data; + } + const instance = target || new ConfigItem(); + instance.configurationScope = data.configurationScope; + instance.description = data.description; + instance.key = data.key; + instance.password = data.password; + instance.value = data.value; + instance.valueType = data.valueType; + return instance; + } +} + export class MessagesInfo { groupId: string; topicName: string; @@ -2191,6 +2214,62 @@ export class MessageCounter { } } +export class MessagingSettings { + acks: number; + batchSize: number; + jmsHost: string; + jmsPort: number; + kafkaHost: string; + kafkaPort: number; + lingerMs: number; + messageMaxBytes: number; + mqttHost: string; + mqttPort: number; + natsHost: string; + natsPort: number; + prioritizedFormats: SpDataFormat[]; + prioritizedProtocols: SpProtocol[]; + pulsarUrl: string; + supportedProtocols: string[]; + zookeeperHost: string; + zookeeperPort: number; + + static fromData( + data: MessagingSettings, + target?: MessagingSettings, + ): MessagingSettings { + if (!data) { + return data; + } + const instance = target || new MessagingSettings(); + instance.acks = data.acks; + instance.batchSize = data.batchSize; + instance.jmsHost = data.jmsHost; + instance.jmsPort = data.jmsPort; + instance.kafkaHost = data.kafkaHost; + instance.kafkaPort = data.kafkaPort; + instance.lingerMs = data.lingerMs; + instance.messageMaxBytes = data.messageMaxBytes; + instance.mqttHost = data.mqttHost; + instance.mqttPort = data.mqttPort; + instance.natsHost = data.natsHost; + instance.natsPort = data.natsPort; + instance.prioritizedFormats = __getCopyArrayFn( + __identity(), + )(data.prioritizedFormats); + instance.prioritizedProtocols = __getCopyArrayFn( + __identity(), + )(data.prioritizedProtocols); + instance.pulsarUrl = data.pulsarUrl; + instance.supportedProtocols = __getCopyArrayFn(__identity())( + data.supportedProtocols, + ); + instance.zookeeperHost = data.zookeeperHost; + instance.zookeeperPort = data.zookeeperPort; + return instance; + } +} + export class MoveRuleDescription extends SchemaTransformationRuleDescription { '@class': 'org.apache.streampipes.model.connect.rules.schema.MoveRuleDescription'; 'newRuntimeKey': string; @@ -3410,6 +3489,77 @@ export class SpQueryResult { } } +export class SpServiceConfiguration { + configs: ConfigItem[]; + rev: string; + serviceGroup: string; + serviceName: string; + + static fromData( + data: SpServiceConfiguration, + target?: SpServiceConfiguration, + ): SpServiceConfiguration { + if (!data) { + return data; + } + const instance = target || new SpServiceConfiguration(); + instance.configs = __getCopyArrayFn(ConfigItem.fromData)(data.configs); + instance.rev = data.rev; + instance.serviceGroup = data.serviceGroup; + instance.serviceName = data.serviceName; + return instance; + } +} + +export class SpServiceRegistration { + firstTimeSeenUnhealthy: number; + healthCheckPath: string; + healthy: boolean; + host: string; + port: number; + rev: string; + scheme: string; + svcGroup: string; + svcId: string; + tags: SpServiceTag[]; + + static fromData( + data: SpServiceRegistration, + target?: SpServiceRegistration, + ): SpServiceRegistration { + if (!data) { + return data; + } + const instance = target || new SpServiceRegistration(); + instance.firstTimeSeenUnhealthy = data.firstTimeSeenUnhealthy; + instance.healthCheckPath = data.healthCheckPath; + instance.healthy = data.healthy; + instance.host = data.host; + instance.port = data.port; + instance.rev = data.rev; + instance.scheme = data.scheme; + instance.svcGroup = data.svcGroup; + instance.svcId = data.svcId; + instance.tags = __getCopyArrayFn(SpServiceTag.fromData)(data.tags); + return instance; + } +} + +export class SpServiceTag { + prefix: SpServiceTagPrefix; + value: string; + + static fromData(data: SpServiceTag, target?: SpServiceTag): SpServiceTag { + if (!data) { + return data; + } + const instance = target || new SpServiceTag(); + instance.prefix = data.prefix; + instance.value = data.value; + return instance; + } +} + export class StaticPropertyAlternative extends StaticProperty { '@class': 'org.apache.streampipes.model.staticproperty.StaticPropertyAlternative'; 'elementId': string; @@ -3818,6 +3968,11 @@ export class WildcardTopicMapping { } } +export type ConfigurationScope = + | 'CONTAINER_STARTUP_CONFIG' + | 'CONTAINER_GLOBAL_CONFIG' + | 'PIPELINE_ELEMENT_CONFIG'; + export type EdgeValidationStatusType = 'COMPLETE' | 'INCOMPLETE' | 'INVALID'; export type EventPropertyUnion = @@ -3847,10 +4002,23 @@ export type SelectionStaticPropertyUnion = | AnyStaticProperty | OneOfStaticProperty; +export type SpDataFormat = 'CBOR' | 'JSON' | 'FST' | 'SMILE'; + export type SpLogLevel = 'INFO' | 'WARN' | 'ERROR'; +export type SpProtocol = 'KAFKA' | 'JMS' | 'MQTT' | 'NATS' | 'PULSAR'; + export type SpQueryStatus = 'OK' | 'TOO_MUCH_DATA'; +export type SpServiceTagPrefix = + | 'SYSTEM' + | 'SP_GROUP' + | 'ADAPTER' + | 'DATA_STREAM' + | 'DATA_PROCESSOR' + | 'DATA_SINK' + | 'DATA_SET'; + export type StaticPropertyType = | 'AnyStaticProperty' | 'CodeInputStaticProperty' diff --git a/ui/src/app/_guards/configured.can-activate.guard.ts b/ui/src/app/_guards/configured.can-activate.guard.ts index fc1e47cb64..6d40db06f5 100644 --- a/ui/src/app/_guards/configured.can-activate.guard.ts +++ b/ui/src/app/_guards/configured.can-activate.guard.ts @@ -32,6 +32,6 @@ export class ConfiguredCanActivateGuard extends BaseConfiguredCanActivateGuard { } onIsUnconfigured(): boolean | UrlTree { - return this.router.parseUrl('setup'); + return this.router.parseUrl('startup'); } } diff --git a/ui/src/app/configuration/configuration-tabs.ts b/ui/src/app/configuration/configuration-tabs.ts index dc4252a511..6977327b35 100644 --- a/ui/src/app/configuration/configuration-tabs.ts +++ b/ui/src/app/configuration/configuration-tabs.ts @@ -36,6 +36,11 @@ export class SpConfigurationTabs { itemTitle: 'Export/Import', itemLink: ['configuration', 'export'], }, + { + itemId: 'extensions-services', + itemTitle: 'Extensions Services', + itemLink: ['configuration', 'extensions-services'], + }, { itemId: 'labels', itemTitle: 'Labels', @@ -51,11 +56,6 @@ export class SpConfigurationTabs { itemTitle: 'Messaging', itemLink: ['configuration', 'messaging'], }, - { - itemId: 'pipelineelement', - itemTitle: 'Pipeline Element Configuration', - itemLink: ['configuration', 'pipelineelement'], - }, { itemId: 'security', itemTitle: 'Security', diff --git a/ui/src/app/configuration/configuration.module.ts b/ui/src/app/configuration/configuration.module.ts index 02a7b9313b..db24cb548d 100644 --- a/ui/src/app/configuration/configuration.module.ts +++ b/ui/src/app/configuration/configuration.module.ts @@ -27,14 +27,8 @@ import { FlexLayoutModule } from '@ngbracket/ngx-layout'; import { CommonModule } from '@angular/common'; import { FormsModule, ReactiveFormsModule } from '@angular/forms'; import { ConfigurationService } from './shared/configuration.service'; -import { ConsulServiceComponent } from './consul-service/consul-service.component'; -import { ConsulConfigsComponent } from './consul-configs/consul-configs.component'; -import { ConsulConfigsTextComponent } from './consul-configs-text/consul-configs-text.component'; -import { ConsulConfigsPasswordComponent } from './consul-configs-password/consul-configs-password.component'; -import { ConsulConfigsBooleanComponent } from './consul-configs-boolean/consul-configs-boolean.component'; -import { ConsulConfigsNumberComponent } from './consul-configs-number/consul-configs-number.component'; + import { CustomMaterialModule } from '../CustomMaterial/custom-material.module'; -import { PipelineElementConfigurationComponent } from './pipeline-element-configuration/pipeline-element-configuration.component'; import { MessagingConfigurationComponent } from './messaging-configuration/messaging-configuration.component'; import { DragDropModule } from '@angular/cdk/drag-drop'; import { DatalakeConfigurationComponent } from './datalake-configuration/datalake-configuration.component'; @@ -61,6 +55,15 @@ import { SpDataExportItemComponent } from './export/export-dialog/data-export-it import { SpEditLabelComponent } from './label-configuration/edit-label/edit-label.component'; import { SpLabelConfigurationComponent } from './label-configuration/label-configuration.component'; import { ColorPickerModule } from 'ngx-color-picker'; +import { ExtensionsServiceManagementComponent } from './extensions-service-management/extensions-service-management.component'; +import { ServiceConfigsComponent } from './extensions-service-management/extensions-service-configuration/service-configs/service-configs.component'; +import { ServiceConfigsTextComponent } from './extensions-service-management/extensions-service-configuration/service-configs/service-configs-text/service-configs-text.component'; +import { ServiceConfigsPasswordComponent } from './extensions-service-management/extensions-service-configuration/service-configs/service-configs-password/service-configs-password.component'; +import { ServiceConfigsBooleanComponent } from './extensions-service-management/extensions-service-configuration/service-configs/service-configs-boolean/service-configs-boolean.component'; +import { ServiceConfigsNumberComponent } from './extensions-service-management/extensions-service-configuration/service-configs/service-configs-number/service-configs-number.component'; +import { SpRegisteredExtensionsServiceComponent } from './extensions-service-management/registrered-extensions-services/registered-extensions-services.component'; +import { SpExtensionsServiceConfigurationComponent } from './extensions-service-management/extensions-service-configuration/extensions-service-configuration.component'; +import { SpMessagingBrokerConfigComponent } from './messaging-configuration/broker-config/broker-config.component'; @NgModule({ imports: [ @@ -114,8 +117,8 @@ import { ColorPickerModule } from 'ngx-color-picker'; component: MessagingConfigurationComponent, }, { - path: 'pipelineelement', - component: PipelineElementConfigurationComponent, + path: 'extensions-services', + component: ExtensionsServiceManagementComponent, }, { path: 'security', @@ -128,18 +131,17 @@ import { ColorPickerModule } from 'ngx-color-picker'; ColorPickerModule, ], declarations: [ - ConsulServiceComponent, - ConsulConfigsComponent, - ConsulConfigsTextComponent, - ConsulConfigsPasswordComponent, - ConsulConfigsBooleanComponent, - ConsulConfigsNumberComponent, + ServiceConfigsComponent, + ServiceConfigsTextComponent, + ServiceConfigsPasswordComponent, + ServiceConfigsBooleanComponent, + ServiceConfigsNumberComponent, DeleteDatalakeIndexComponent, EditUserDialogComponent, EditGroupDialogComponent, EmailConfigurationComponent, GeneralConfigurationComponent, - PipelineElementConfigurationComponent, + ExtensionsServiceManagementComponent, SecurityAuthenticationConfigurationComponent, SecurityConfigurationComponent, SecurityUserConfigComponent, @@ -153,6 +155,9 @@ import { ColorPickerModule } from 'ngx-color-picker'; SpDataImportDialogComponent, SpEditLabelComponent, SpLabelConfigurationComponent, + SpMessagingBrokerConfigComponent, + SpRegisteredExtensionsServiceComponent, + SpExtensionsServiceConfigurationComponent, ], providers: [ConfigurationService], }) diff --git a/ui/src/app/configuration/consul-configs-boolean/consul-configs-boolean.component.css b/ui/src/app/configuration/consul-configs-boolean/consul-configs-boolean.component.css deleted file mode 100644 index 13cbc4aacb..0000000000 --- a/ui/src/app/configuration/consul-configs-boolean/consul-configs-boolean.component.css +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ diff --git a/ui/src/app/configuration/consul-configs-text/consul-configs-text.component.css b/ui/src/app/configuration/consul-configs-text/consul-configs-text.component.css deleted file mode 100644 index 13cbc4aacb..0000000000 --- a/ui/src/app/configuration/consul-configs-text/consul-configs-text.component.css +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ diff --git a/ui/src/app/configuration/consul-configs/consul-configs.component.html b/ui/src/app/configuration/consul-configs/consul-configs.component.html deleted file mode 100644 index 5125ce00bd..0000000000 --- a/ui/src/app/configuration/consul-configs/consul-configs.component.html +++ /dev/null @@ -1,73 +0,0 @@ - - -
-
- - - - - - - - - - - - - - - -
- -
-
-
diff --git a/ui/src/app/configuration/consul-service/consul-service.component.html b/ui/src/app/configuration/consul-service/consul-service.component.html deleted file mode 100644 index 13fe34293a..0000000000 --- a/ui/src/app/configuration/consul-service/consul-service.component.html +++ /dev/null @@ -1,58 +0,0 @@ - - -
- - lens - - - lens - -

{{ consulService.name }}

- - expand_less - - - expand_more - -
- -
- -
diff --git a/ui/src/app/configuration/consul-service/consul-service.component.ts b/ui/src/app/configuration/consul-service/consul-service.component.ts deleted file mode 100644 index ef595de7b6..0000000000 --- a/ui/src/app/configuration/consul-service/consul-service.component.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import { Component, EventEmitter, Input, Output } from '@angular/core'; -import { StreampipesPeContainer } from '../shared/streampipes-pe-container.model'; - -@Component({ - selector: 'sp-consul-service', - templateUrl: './consul-service.component.html', - styleUrls: ['./consul-service.component.css'], -}) -export class ConsulServiceComponent { - @Input() consulService: StreampipesPeContainer; - @Output() updateConsulService: EventEmitter = - new EventEmitter(); - showConfiguration = false; - - constructor() {} - - toggleConfiguration(): void { - this.showConfiguration = !this.showConfiguration; - } - - updateConfiguration(): void { - this.updateConsulService.emit(this.consulService); - } -} diff --git a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.ts b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.ts index 783f8b0937..5a0b63bfaa 100644 --- a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.ts +++ b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.ts @@ -20,12 +20,10 @@ import { Component, OnInit, ViewChild } from '@angular/core'; import { MatTableDataSource } from '@angular/material/table'; import { DataLakeConfigurationEntry } from './datalake-configuration-entry'; import { - DataExplorerDataConfig, DatalakeQueryParameterBuilder, DatalakeQueryParameters, DatalakeRestService, DataViewDataExplorerService, - DateRange, EventSchema, FieldConfig, SpQueryResult, diff --git a/ui/src/app/configuration/email-configuration/email-configuration.component.ts b/ui/src/app/configuration/email-configuration/email-configuration.component.ts index 7bd12f433d..1013d0d653 100644 --- a/ui/src/app/configuration/email-configuration/email-configuration.component.ts +++ b/ui/src/app/configuration/email-configuration/email-configuration.component.ts @@ -56,7 +56,7 @@ export class EmailConfigurationComponent implements OnInit { ngOnInit(): void { this.breadcrumbService.updateBreadcrumb([ SpConfigurationRoutes.BASE, - { label: SpConfigurationTabs.getTabs()[2].itemTitle }, + { label: SpConfigurationTabs.getTabs()[5].itemTitle }, ]); this.loadMailConfig(true); } diff --git a/ui/src/app/configuration/export/data-export.service.ts b/ui/src/app/configuration/export/data-export.service.ts index 5fe23cf692..61082b7612 100644 --- a/ui/src/app/configuration/export/data-export.service.ts +++ b/ui/src/app/configuration/export/data-export.service.ts @@ -19,8 +19,8 @@ import { Injectable } from '@angular/core'; import { AssetExportConfiguration, - PlatformServicesCommons, ExportConfiguration, + PlatformServicesCommons, } from '@streampipes/platform-services'; import { HttpClient, diff --git a/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/extensions-service-configuration.component.html b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/extensions-service-configuration.component.html new file mode 100644 index 0000000000..18dd75698d --- /dev/null +++ b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/extensions-service-configuration.component.html @@ -0,0 +1,127 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + +
+ Service Name + + {{ element.serviceGroup }} + + Service Name + + {{ element.serviceName }} + + Action + + + +
+
+ +
+
+
+
diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/ISpKvManagement.java b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/extensions-service-configuration.component.scss similarity index 74% rename from streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/ISpKvManagement.java rename to ui/src/app/configuration/extensions-service-management/extensions-service-configuration/extensions-service-configuration.component.scss index 4b859f3c67..5fc34db9de 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/ISpKvManagement.java +++ b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/extensions-service-configuration.component.scss @@ -1,4 +1,4 @@ -/* +/*! * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -15,16 +15,27 @@ * limitations under the License. * */ -package org.apache.streampipes.svcdiscovery.api; -import java.util.Map; +table { + width: 100%; +} -public interface ISpKvManagement { +tr.consul-configuration-detail-row { + height: 0; +} - Map getKeyValue(String route); +.consul-configuration-row td { + border-bottom-width: 0; +} - void updateConfig(String key, String entry, boolean password); +.consul-configuration-detail { + overflow: hidden; + display: flex; +} - void deleteConfig(String key); +.mat-table { +} +.consul-service { + padding: 5px; } diff --git a/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/extensions-service-configuration.component.ts b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/extensions-service-configuration.component.ts new file mode 100644 index 0000000000..38bdad6055 --- /dev/null +++ b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/extensions-service-configuration.component.ts @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Component, ViewChild } from '@angular/core'; +import { MatPaginator } from '@angular/material/paginator'; +import { MatTableDataSource } from '@angular/material/table'; +import { ConfigurationService } from '../../shared/configuration.service'; +import { + animate, + state, + style, + transition, + trigger, +} from '@angular/animations'; +import { SpServiceConfiguration } from '@streampipes/platform-services'; + +@Component({ + selector: 'sp-extensions-service-configuration', + templateUrl: './extensions-service-configuration.component.html', + styleUrls: ['./extensions-service-configuration.component.scss'], + animations: [ + trigger('detailExpand', [ + state( + 'collapsed', + style({ height: '0px', minHeight: '0', display: 'none' }), + ), + state('expanded', style({ height: '*' })), + transition( + 'expanded <=> collapsed', + animate('225ms cubic-bezier(0.4, 0.0, 0.2, 1)'), + ), + ]), + ], +}) +export class SpExtensionsServiceConfigurationComponent { + displayedColumns: string[] = ['group', 'name', 'action']; + @ViewChild(MatPaginator, { static: false }) paginator: MatPaginator; + dataSource = new MatTableDataSource(); + + expandedElement: any; + consulServices: SpServiceConfiguration[]; + + constructor(private configurationService: ConfigurationService) { + this.getConsulServices(); + } + + getConsulServices(): void { + this.configurationService.getExtensionsServiceConfigs().subscribe( + response => { + const sortedServices = this.sort(response); + this.consulServices = sortedServices; + this.dataSource.data = sortedServices; + console.log(this.consulServices); + }, + error => { + console.error(error); + }, + ); + } + + sort(consulServices: SpServiceConfiguration[]): SpServiceConfiguration[] { + if (!consulServices || consulServices.length === 0) { + return null; + } + + consulServices.sort( + (a: SpServiceConfiguration, b: SpServiceConfiguration) => { + if (a.serviceGroup < b.serviceGroup) { + return -1; + } else if (a.serviceGroup > b.serviceGroup) { + return 1; + } else { + return 0; + } + }, + ); + return consulServices; + } + + updateConsulService(config: SpServiceConfiguration): void { + this.configurationService + .updateExtensionsServiceConfigs(config) + .subscribe( + () => { + this.getConsulServices(); + }, + error => { + console.error(error); + }, + ); + } + + expand(element: SpServiceConfiguration) { + if (this.expandedElement === element) { + this.expandedElement = undefined; + } else { + this.expandedElement = element; + } + } +} diff --git a/ui/src/app/configuration/consul-configs-boolean/consul-configs-boolean.component.html b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-boolean/service-configs-boolean.component.html similarity index 100% rename from ui/src/app/configuration/consul-configs-boolean/consul-configs-boolean.component.html rename to ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-boolean/service-configs-boolean.component.html diff --git a/ui/src/app/configuration/consul-configs-boolean/consul-configs-boolean.component.ts b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-boolean/service-configs-boolean.component.ts similarity index 72% rename from ui/src/app/configuration/consul-configs-boolean/consul-configs-boolean.component.ts rename to ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-boolean/service-configs-boolean.component.ts index 856edc6837..cf69fa2e33 100644 --- a/ui/src/app/configuration/consul-configs-boolean/consul-configs-boolean.component.ts +++ b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-boolean/service-configs-boolean.component.ts @@ -17,16 +17,15 @@ */ import { Component, Input } from '@angular/core'; -import { StreampipesPeContainerConifgs } from '../shared/streampipes-pe-container-configs'; -import { ConfigurationService } from '../shared/configuration.service'; +import { ConfigurationService } from '../../../../shared/configuration.service'; +import { ConfigItem } from '@streampipes/platform-services'; @Component({ selector: 'sp-consul-configs-boolean', - templateUrl: './consul-configs-boolean.component.html', - styleUrls: ['./consul-configs-boolean.component.css'], + templateUrl: './service-configs-boolean.component.html', providers: [ConfigurationService], }) -export class ConsulConfigsBooleanComponent { - @Input() configuration: StreampipesPeContainerConifgs; +export class ServiceConfigsBooleanComponent { + @Input() configuration: ConfigItem; constructor(public configService: ConfigurationService) {} } diff --git a/ui/src/app/configuration/consul-configs-number/consul-configs-number.component.html b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-number/service-configs-number.component.html similarity index 94% rename from ui/src/app/configuration/consul-configs-number/consul-configs-number.component.html rename to ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-number/service-configs-number.component.html index 2ea4ba0227..d0220e7838 100644 --- a/ui/src/app/configuration/consul-configs-number/consul-configs-number.component.html +++ b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-number/service-configs-number.component.html @@ -16,7 +16,7 @@ ~ --> - + - + {{ configuration.key }} { - let fixture: ComponentFixture; + let fixture: ComponentFixture; let configurationServiceStub: Partial; - let component: ConsulConfigsPasswordComponent; + let component: ServiceConfigsPasswordComponent; let configurationServcie: ConfigurationService; @@ -68,7 +68,7 @@ describe('ConsulConfigsPasswordComponent', () => { FormsModule, HttpClientTestingModule, ], - declarations: [ConsulConfigsPasswordComponent], + declarations: [ServiceConfigsPasswordComponent], providers: [ { provide: ConfigurationService, @@ -77,7 +77,7 @@ describe('ConsulConfigsPasswordComponent', () => { ], }).compileComponents(); - fixture = TestBed.createComponent(ConsulConfigsPasswordComponent); + fixture = TestBed.createComponent(ServiceConfigsPasswordComponent); configurationServcie = fixture.debugElement.injector.get(ConfigurationService); diff --git a/ui/src/app/configuration/consul-configs-password/consul-configs-password.component.ts b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-password/service-configs-password.component.ts similarity index 80% rename from ui/src/app/configuration/consul-configs-password/consul-configs-password.component.ts rename to ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-password/service-configs-password.component.ts index 0923c2f938..cccf0d7e73 100644 --- a/ui/src/app/configuration/consul-configs-password/consul-configs-password.component.ts +++ b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-password/service-configs-password.component.ts @@ -17,19 +17,19 @@ */ import { Component, Input } from '@angular/core'; -import { StreampipesPeContainerConifgs } from '../shared/streampipes-pe-container-configs'; -import { ConfigurationService } from '../shared/configuration.service'; +import { ConfigurationService } from '../../../../shared/configuration.service'; +import { ConfigItem } from '@streampipes/platform-services'; const hiddenPasswordString = '*****'; @Component({ selector: 'sp-consul-configs-password', - templateUrl: './consul-configs-password.component.html', - styleUrls: ['./consul-configs-password.component.css'], + templateUrl: './service-configs-password.component.html', + styleUrls: ['./service-configs-password.component.css'], providers: [ConfigurationService], }) -export class ConsulConfigsPasswordComponent { - @Input() configuration: StreampipesPeContainerConifgs; +export class ServiceConfigsPasswordComponent { + @Input() configuration: ConfigItem; password: string; show: boolean; diff --git a/ui/src/app/configuration/consul-configs-text/consul-configs-text.component.html b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-text/service-configs-text.component.html similarity index 95% rename from ui/src/app/configuration/consul-configs-text/consul-configs-text.component.html rename to ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-text/service-configs-text.component.html index 5a8894efaa..9b9b56c7a8 100644 --- a/ui/src/app/configuration/consul-configs-text/consul-configs-text.component.html +++ b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs-text/service-configs-text.component.html @@ -16,7 +16,7 @@ ~ --> - + + +
+
+
+ + + + + + + + + + + +
+
+ +
+
+
diff --git a/ui/src/app/configuration/consul-configs/consul-configs.component.ts b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs.component.ts similarity index 71% rename from ui/src/app/configuration/consul-configs/consul-configs.component.ts rename to ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs.component.ts index b260d2fc05..8bfce5ba12 100644 --- a/ui/src/app/configuration/consul-configs/consul-configs.component.ts +++ b/ui/src/app/configuration/extensions-service-management/extensions-service-configuration/service-configs/service-configs.component.ts @@ -17,19 +17,18 @@ */ import { Component, EventEmitter, Input, Output } from '@angular/core'; -import { StreampipesPeContainer } from '../shared/streampipes-pe-container.model'; -import { XsService } from '../../NS/xs.service'; +import { XsService } from '../../../../NS/xs.service'; +import { SpServiceConfiguration } from '@streampipes/platform-services'; @Component({ selector: 'sp-consul-configs', - templateUrl: './consul-configs.component.html', - styleUrls: ['./consul-configs.component.css'], + templateUrl: './service-configs.component.html', providers: [XsService], }) -export class ConsulConfigsComponent { - @Input() consulService: StreampipesPeContainer; - @Output() updateConsulService: EventEmitter = - new EventEmitter(); +export class ServiceConfigsComponent { + @Input() consulService: SpServiceConfiguration; + @Output() updateConsulService: EventEmitter = + new EventEmitter(); constructor(private service: XsService) {} diff --git a/ui/src/app/configuration/extensions-service-management/extensions-service-management.component.html b/ui/src/app/configuration/extensions-service-management/extensions-service-management.component.html new file mode 100644 index 0000000000..3958c34611 --- /dev/null +++ b/ui/src/app/configuration/extensions-service-management/extensions-service-management.component.html @@ -0,0 +1,39 @@ + + + +
+
+ + + + + + +
+
+ diff --git a/ui/src/app/configuration/extensions-service-management/extensions-service-management.component.ts b/ui/src/app/configuration/extensions-service-management/extensions-service-management.component.ts new file mode 100644 index 0000000000..13190bd02f --- /dev/null +++ b/ui/src/app/configuration/extensions-service-management/extensions-service-management.component.ts @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Component, OnInit } from '@angular/core'; +import { SpConfigurationTabs } from '../configuration-tabs'; +import { SpBreadcrumbService } from '@streampipes/shared-ui'; +import { SpConfigurationRoutes } from '../configuration.routes'; + +@Component({ + selector: 'sp-extensions-service-management', + templateUrl: './extensions-service-management.component.html', +}) +export class ExtensionsServiceManagementComponent implements OnInit { + tabs = SpConfigurationTabs.getTabs(); + + constructor(private breadcrumbService: SpBreadcrumbService) {} + + ngOnInit() { + this.breadcrumbService.updateBreadcrumb([ + SpConfigurationRoutes.BASE, + { label: SpConfigurationTabs.getTabs()[3].itemTitle }, + ]); + } +} diff --git a/ui/src/app/configuration/extensions-service-management/registrered-extensions-services/registered-extensions-services.component.html b/ui/src/app/configuration/extensions-service-management/registrered-extensions-services/registered-extensions-services.component.html new file mode 100644 index 0000000000..8cea7ddc60 --- /dev/null +++ b/ui/src/app/configuration/extensions-service-management/registrered-extensions-services/registered-extensions-services.component.html @@ -0,0 +1,84 @@ + + +
+ + + + + + + + + + + + + + +
+ Status + + + lens + + + lens + + + Service ID + + {{ element.svcId }} +
+
diff --git a/ui/src/app/configuration/consul-configs/consul-configs.component.css b/ui/src/app/configuration/extensions-service-management/registrered-extensions-services/registered-extensions-services.component.scss similarity index 85% rename from ui/src/app/configuration/consul-configs/consul-configs.component.css rename to ui/src/app/configuration/extensions-service-management/registrered-extensions-services/registered-extensions-services.component.scss index 13cbc4aacb..c75dfb8bca 100644 --- a/ui/src/app/configuration/consul-configs/consul-configs.component.css +++ b/ui/src/app/configuration/extensions-service-management/registrered-extensions-services/registered-extensions-services.component.scss @@ -15,3 +15,13 @@ * limitations under the License. * */ + +.service-icon-passing { + cursor: default; + color: #4caf50; +} + +.service-icon-critical { + cursor: default; + color: #f44336; +} diff --git a/ui/src/app/configuration/extensions-service-management/registrered-extensions-services/registered-extensions-services.component.ts b/ui/src/app/configuration/extensions-service-management/registrered-extensions-services/registered-extensions-services.component.ts new file mode 100644 index 0000000000..cb1f3a50cf --- /dev/null +++ b/ui/src/app/configuration/extensions-service-management/registrered-extensions-services/registered-extensions-services.component.ts @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Component, ViewChild } from '@angular/core'; +import { MatPaginator } from '@angular/material/paginator'; +import { MatTableDataSource } from '@angular/material/table'; +import { SpServiceRegistration } from '@streampipes/platform-services'; +import { ConfigurationService } from '../../shared/configuration.service'; + +@Component({ + selector: 'sp-registered-extensions-services', + templateUrl: './registered-extensions-services.component.html', + styleUrls: ['./registered-extensions-services.component.scss'], +}) +export class SpRegisteredExtensionsServiceComponent { + displayedColumns: string[] = ['status', 'name']; + @ViewChild(MatPaginator, { static: false }) paginator: MatPaginator; + dataSource = new MatTableDataSource(); + + constructor(private configurationService: ConfigurationService) { + this.getRegisteredServices(); + } + + getRegisteredServices(): void { + this.configurationService + .getRegisteredExtensionsServices() + .subscribe(res => { + this.dataSource.data = res; + }); + } +} diff --git a/ui/src/app/configuration/messaging-configuration/broker-config/broker-config.component.html b/ui/src/app/configuration/messaging-configuration/broker-config/broker-config.component.html new file mode 100644 index 0000000000..d97c4bd56b --- /dev/null +++ b/ui/src/app/configuration/messaging-configuration/broker-config/broker-config.component.html @@ -0,0 +1,35 @@ + + +
+
{{ title }}
+
+
+ + Host + + +
+
+ + Port + + +
+
+
diff --git a/streampipes-extensions/streampipes-pipeline-elements-all-jvm/src/main/java/org/apache/streampipes/pe/jvm/config/ConfigKeys.java b/ui/src/app/configuration/messaging-configuration/broker-config/broker-config.component.ts similarity index 72% rename from streampipes-extensions/streampipes-pipeline-elements-all-jvm/src/main/java/org/apache/streampipes/pe/jvm/config/ConfigKeys.java rename to ui/src/app/configuration/messaging-configuration/broker-config/broker-config.component.ts index 77412c1b7f..9a00398bb4 100644 --- a/streampipes-extensions/streampipes-pipeline-elements-all-jvm/src/main/java/org/apache/streampipes/pe/jvm/config/ConfigKeys.java +++ b/ui/src/app/configuration/messaging-configuration/broker-config/broker-config.component.ts @@ -15,10 +15,20 @@ * limitations under the License. * */ -package org.apache.streampipes.pe.jvm.config; -public class ConfigKeys { - static final String HOST = "SP_HOST"; - static final String PORT = "SP_PORT"; - static final String SERVICE_NAME_KEY = "SP_SERVICE_NAME"; +import { Component, Input } from '@angular/core'; + +@Component({ + selector: 'sp-messaging-broker-config', + templateUrl: './broker-config.component.html', +}) +export class SpMessagingBrokerConfigComponent { + @Input() + title: string; + + @Input() + host: string; + + @Input() + port: number; } diff --git a/ui/src/app/configuration/messaging-configuration/messaging-configuration.component.html b/ui/src/app/configuration/messaging-configuration/messaging-configuration.component.html index 7afdeffa6f..8f96bc8864 100644 --- a/ui/src/app/configuration/messaging-configuration/messaging-configuration.component.html +++ b/ui/src/app/configuration/messaging-configuration/messaging-configuration.component.html @@ -167,5 +167,58 @@ + + + + + + +
+
Pulsar
+ + URL + + +
+ +
+ +
+ diff --git a/ui/src/app/configuration/messaging-configuration/messaging-configuration.component.ts b/ui/src/app/configuration/messaging-configuration/messaging-configuration.component.ts index 918421442e..ec9c7d67aa 100644 --- a/ui/src/app/configuration/messaging-configuration/messaging-configuration.component.ts +++ b/ui/src/app/configuration/messaging-configuration/messaging-configuration.component.ts @@ -18,11 +18,11 @@ import { Component, OnInit } from '@angular/core'; import { ConfigurationService } from '../shared/configuration.service'; -import { MessagingSettings } from '../shared/messaging-settings.model'; import { CdkDragDrop, moveItemInArray } from '@angular/cdk/drag-drop'; import { SpConfigurationTabs } from '../configuration-tabs'; import { SpBreadcrumbService } from '@streampipes/shared-ui'; import { SpConfigurationRoutes } from '../configuration.routes'; +import { MessagingSettings } from '@streampipes/platform-services'; @Component({ selector: 'sp-messaging-configuration', @@ -43,7 +43,7 @@ export class MessagingConfigurationComponent implements OnInit { ngOnInit() { this.breadcrumbService.updateBreadcrumb([ SpConfigurationRoutes.BASE, - { label: SpConfigurationTabs.getTabs()[4].itemTitle }, + { label: SpConfigurationTabs.getTabs()[6].itemTitle }, ]); this.getMessagingSettings(); } diff --git a/ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.html b/ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.html deleted file mode 100644 index ec655e0ebc..0000000000 --- a/ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.html +++ /dev/null @@ -1,155 +0,0 @@ - - - -
-
- - - - - - - - - - - - - - - - - - - - - - - - -
- Status - - - lens - - - lens - - - Extensions Service Group - - {{ element.name }} - - Action - - - -
-
- -
-
-
- -
-
- diff --git a/ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.ts b/ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.ts deleted file mode 100644 index ff5f5d670c..0000000000 --- a/ui/src/app/configuration/pipeline-element-configuration/pipeline-element-configuration.component.ts +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import { Component, OnInit, ViewChild } from '@angular/core'; -import { - animate, - state, - style, - transition, - trigger, -} from '@angular/animations'; -import { ConfigurationService } from '../shared/configuration.service'; -import { StreampipesPeContainer } from '../shared/streampipes-pe-container.model'; -import { StreampipesPeContainerConifgs } from '../shared/streampipes-pe-container-configs'; -import { MatPaginator } from '@angular/material/paginator'; -import { MatTableDataSource } from '@angular/material/table'; -import { SpConfigurationTabs } from '../configuration-tabs'; -import { SpBreadcrumbService } from '@streampipes/shared-ui'; -import { SpConfigurationRoutes } from '../configuration.routes'; - -@Component({ - selector: 'sp-pipeline-element-configuration', - templateUrl: './pipeline-element-configuration.component.html', - styleUrls: ['./pipeline-element-configuration.component.css'], - animations: [ - trigger('detailExpand', [ - state( - 'collapsed', - style({ height: '0px', minHeight: '0', display: 'none' }), - ), - state('expanded', style({ height: '*' })), - transition( - 'expanded <=> collapsed', - animate('225ms cubic-bezier(0.4, 0.0, 0.2, 1)'), - ), - ]), - ], -}) -export class PipelineElementConfigurationComponent implements OnInit { - tabs = SpConfigurationTabs.getTabs(); - - consulServices: StreampipesPeContainer[]; - - displayedColumns: string[] = ['status', 'name', 'action']; - @ViewChild(MatPaginator, { static: false }) paginator: MatPaginator; - dataSource = new MatTableDataSource(); - - expandedElement: any; - - selectedConsulService: StreampipesPeContainer; - consulServiceSelected = false; - - constructor( - private configurationService: ConfigurationService, - private breadcrumbService: SpBreadcrumbService, - ) { - this.getConsulServices(); - } - - ngOnInit() { - this.breadcrumbService.updateBreadcrumb([ - SpConfigurationRoutes.BASE, - { label: SpConfigurationTabs.getTabs()[5].itemTitle }, - ]); - } - - getConsulServices(): void { - this.configurationService.getConsulServices().subscribe( - response => { - const sortedServices = this.sort(response); - this.consulServices = sortedServices; - this.dataSource.data = sortedServices; - }, - error => { - console.error(error); - }, - ); - } - - sort(consulServices: StreampipesPeContainer[]): StreampipesPeContainer[] { - if (!consulServices || consulServices.length === 0) { - return null; - } - - consulServices.sort( - (a: StreampipesPeContainer, b: StreampipesPeContainer) => { - if (a.name < b.name) { - return -1; - } else if (a.name > b.name) { - return 1; - } else { - return 0; - } - }, - ); - consulServices.forEach(cs => - cs.configs.sort( - ( - a: StreampipesPeContainerConifgs, - b: StreampipesPeContainerConifgs, - ) => { - if (a.key < b.key) { - return -1; - } else if (a.key > b.key) { - return 1; - } else { - return 0; - } - }, - ), - ); - return consulServices; - } - - updateConsulService(consulService: StreampipesPeContainer): void { - this.configurationService.updateConsulService(consulService).subscribe( - response => {}, - error => { - console.error(error); - }, - ); - } - - expand(element: StreampipesPeContainer) { - if (this.expandedElement === element) { - this.expandedElement = undefined; - } else { - this.expandedElement = element; - } - } -} diff --git a/ui/src/app/configuration/security-configuration/abstract-security-principal-config.ts b/ui/src/app/configuration/security-configuration/abstract-security-principal-config.ts index 7bb388d2ea..998f9e8e1f 100644 --- a/ui/src/app/configuration/security-configuration/abstract-security-principal-config.ts +++ b/ui/src/app/configuration/security-configuration/abstract-security-principal-config.ts @@ -23,8 +23,8 @@ import { MatTableDataSource } from '@angular/material/table'; import { ServiceAccount, UserAccount, - UserService, UserAdminService, + UserService, } from '@streampipes/platform-services'; import { Observable } from 'rxjs'; import { DialogService, PanelType } from '@streampipes/shared-ui'; diff --git a/ui/src/app/configuration/security-configuration/edit-user-dialog/edit-user-dialog.component.ts b/ui/src/app/configuration/security-configuration/edit-user-dialog/edit-user-dialog.component.ts index a8a268b183..03fa4641bd 100644 --- a/ui/src/app/configuration/security-configuration/edit-user-dialog/edit-user-dialog.component.ts +++ b/ui/src/app/configuration/security-configuration/edit-user-dialog/edit-user-dialog.component.ts @@ -20,12 +20,12 @@ import { Component, Input, OnInit, ViewEncapsulation } from '@angular/core'; import { CurrentUserService, DialogRef } from '@streampipes/shared-ui'; import { Group, + MailConfigService, Role, ServiceAccount, UserAccount, - MailConfigService, - UserService, UserGroupService, + UserService, } from '@streampipes/platform-services'; import { AbstractControl, diff --git a/ui/src/app/configuration/security-configuration/security-configuration.component.ts b/ui/src/app/configuration/security-configuration/security-configuration.component.ts index 5b9670d8f2..a0ad1c0643 100644 --- a/ui/src/app/configuration/security-configuration/security-configuration.component.ts +++ b/ui/src/app/configuration/security-configuration/security-configuration.component.ts @@ -34,7 +34,7 @@ export class SecurityConfigurationComponent implements OnInit { ngOnInit(): void { this.breadcrumbService.updateBreadcrumb([ SpConfigurationRoutes.BASE, - { label: SpConfigurationTabs.getTabs()[6].itemTitle }, + { label: SpConfigurationTabs.getTabs()[7].itemTitle }, ]); } } diff --git a/ui/src/app/configuration/security-configuration/user-group-configuration/user-group-configuration.component.ts b/ui/src/app/configuration/security-configuration/user-group-configuration/user-group-configuration.component.ts index 24f05e9486..d4f23cf61e 100644 --- a/ui/src/app/configuration/security-configuration/user-group-configuration/user-group-configuration.component.ts +++ b/ui/src/app/configuration/security-configuration/user-group-configuration/user-group-configuration.component.ts @@ -21,7 +21,7 @@ import { Group, UserGroupService } from '@streampipes/platform-services'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; import { MatTableDataSource } from '@angular/material/table'; -import { PanelType, DialogService } from '@streampipes/shared-ui'; +import { DialogService, PanelType } from '@streampipes/shared-ui'; import { EditGroupDialogComponent } from '../edit-group-dialog/edit-group-dialog.component'; @Component({ diff --git a/ui/src/app/configuration/shared/configuration.service.spec.ts b/ui/src/app/configuration/shared/configuration.service.spec.ts index 7acc855c3e..c1138d8d84 100644 --- a/ui/src/app/configuration/shared/configuration.service.spec.ts +++ b/ui/src/app/configuration/shared/configuration.service.spec.ts @@ -22,7 +22,7 @@ import { HttpTestingController, } from '@angular/common/http/testing'; import { ConfigurationService } from './configuration.service'; -import { StreampipesPeContainer } from './streampipes-pe-container.model'; +import { SpServiceConfiguration } from '../../../../projects/streampipes/platform-services/src/lib/model/gen/streampipes-model'; describe('ConfigurationService', () => { let injector: TestBed; @@ -42,18 +42,22 @@ describe('ConfigurationService', () => { httpMock.verify(); }); - it('should create Get to /api/v2/consul', () => { - service.getConsulServices().subscribe(res => res); - const req = httpMock.expectOne('/streampipes-backend/api/v2/consul'); + it('should create Get to /api/v2/extensions-services-configurations', () => { + service.getExtensionsServiceConfigs().subscribe(res => res); + const req = httpMock.expectOne( + '/streampipes-backend/api/v2/extensions-services-configurations', + ); expect(req.request.method).toBe('GET'); }); - it('should create Post to /api/v2/consul', () => { + it('should create Put to /api/v2/extensions-services-configurations/abc', () => { service - .updateConsulService({} as StreampipesPeContainer) + .updateExtensionsServiceConfigs({} as SpServiceConfiguration) .subscribe(res => res); - const req = httpMock.expectOne('/streampipes-backend/api/v2/consul'); - expect(req.request.method).toBe('POST'); + const req = httpMock.expectOne( + '/streampipes-backend/api/v2/extensions-services-configurations/abc', + ); + expect(req.request.method).toBe('PUT'); }); it('should get Server URL', () => { diff --git a/ui/src/app/configuration/shared/configuration.service.ts b/ui/src/app/configuration/shared/configuration.service.ts index fdaea5c192..2e71d80371 100644 --- a/ui/src/app/configuration/shared/configuration.service.ts +++ b/ui/src/app/configuration/shared/configuration.service.ts @@ -20,10 +20,12 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { Observable } from 'rxjs'; import { map } from 'rxjs/operators'; - -import { StreampipesPeContainer } from './streampipes-pe-container.model'; -import { MessagingSettings } from './messaging-settings.model'; import { MultipartUtils } from './multipart-utils'; +import { + MessagingSettings, + SpServiceConfiguration, + SpServiceRegistration, +} from '@streampipes/platform-services'; @Injectable() export class ConfigurationService { @@ -49,40 +51,43 @@ export class ConfigurationService { } getMessagingSettings(): Observable { + return this.http.get(this.getServerUrl() + '/api/v2/messaging').pipe( + map(response => { + return response as MessagingSettings; + }), + ); + } + + getRegisteredExtensionsServices(): Observable { return this.http - .get(this.getServerUrl() + '/api/v2/consul/messaging') + .get(this.getServerUrl() + '/api/v2/extensions-services') .pipe( map(response => { - return response as MessagingSettings; + return response as SpServiceRegistration[]; }), ); } - getConsulServices(): Observable { - return this.http.get(this.getServerUrl() + '/api/v2/consul').pipe( - map(response => { - for (const service of response as any[]) { - for (const config of service['configs']) { - if (config.valueType === 'xs:integer') { - config.value = parseInt(config.value); - } else if (config.valueType === 'xs:double') { - config.value = parseFloat('xs:double'); - } else if (config.valueType === 'xs:boolean') { - config.value = config.value === 'true'; - } - } - } - return response as StreampipesPeContainer[]; - }), - ); + getExtensionsServiceConfigs(): Observable { + return this.http + .get( + this.getServerUrl() + + '/api/v2/extensions-services-configurations', + ) + .pipe( + map(response => { + return response as SpServiceConfiguration[]; + }), + ); } - updateConsulService( - consulService: StreampipesPeContainer, + updateExtensionsServiceConfigs( + config: SpServiceConfiguration, ): Observable { - return this.http.post( - this.getServerUrl() + '/api/v2/consul', - consulService, + return this.http.put( + this.getServerUrl() + + `/api/v2/extensions-services-configurations/${config.serviceGroup}`, + config, ); } @@ -90,7 +95,7 @@ export class ConfigurationService { messagingSettings: MessagingSettings, ): Observable { return this.http.post( - this.getServerUrl() + '/api/v2/consul/messaging', + this.getServerUrl() + '/api/v2/messaging', messagingSettings, ); } diff --git a/ui/src/app/configuration/shared/configuration.test.service.ts b/ui/src/app/configuration/shared/configuration.test.service.ts deleted file mode 100644 index b762f7cc87..0000000000 --- a/ui/src/app/configuration/shared/configuration.test.service.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import { Injectable } from '@angular/core'; -import { Observable, of } from 'rxjs'; - -import { StreampipesPeContainer } from './streampipes-pe-container.model'; -import { StreampipesPeContainerConifgs } from './streampipes-pe-container-configs'; - -@Injectable() -export class ConfigurationMockService { - constructor() {} - - getServerUrl() { - return '/streampipes-backend'; - } - - getConsulServices(): Observable { - const config: StreampipesPeContainerConifgs[] = []; - config[0] = new StreampipesPeContainerConifgs(); - config[0].description = 'test int'; - config[0].key = 'testint'; - config[0].value = '80'; - config[0].valueType = 'xs:integer'; - config[0].isPassword = false; - - return of([ - { - mainKey: 'sp/test/1', - meta: { - status: 'passing', - }, - name: 'test1', - configs: [ - config[0], - { - key: 'teststring', - value: '765', - valueType: 'xs:string', - isPassword: false, - description: 'test string', - }, - ], - }, - { - mainKey: 'sp/test/2', - meta: { - status: 'critical', - }, - name: 'test2', - configs: [ - { - key: 'testbool', - value: 'false', - valueType: 'xs:boolean', - isPassword: false, - description: 'test bool', - }, - ], - }, - ] as StreampipesPeContainer[]); - } - - updateConsulService( - consulService: StreampipesPeContainer, - ): Observable { - return of({}); - } -} diff --git a/ui/src/app/login/components/startup/startup.component.ts b/ui/src/app/login/components/startup/startup.component.ts index 8a9833dc53..5b710cd6bd 100644 --- a/ui/src/app/login/components/startup/startup.component.ts +++ b/ui/src/app/login/components/startup/startup.component.ts @@ -46,8 +46,13 @@ export class StartupComponent implements OnInit { this.authService.checkConfiguration().subscribe( configured => { this.progress = 100; - const target: string = configured ? 'login' : 'setup'; - this.router.navigate([target]); + if (configured) { + this.router.navigate(['login']); + } else { + setTimeout(() => { + this.checkStatus(); + }, this.loadingIntervalInSeconds * 1000); + } }, () => { this.currentStep += this.loadingIntervalInSeconds;