extend CLI by monitoring stack (#2256)

* chore: extend CLI by monitoring stack

* chore: extend dependabot config by new docker images

* chore: add env variable to backend & extensions service container

.github/dependabot.yml

@@ -22,7 +22,6 @@ updates:
     schedule:
       interval: "daily"
     open-pull-requests-limit: 1
-
   - package-ecosystem: "docker"
     directory: "/installer/compose"
     labels:
@@ -31,7 +30,6 @@ updates:
     schedule:
       interval: "daily"
     open-pull-requests-limit: 1
-
   - package-ecosystem: "docker"
     directory: "/installer/cli/deploy/standalone/grafana"
     labels:
@@ -40,6 +38,22 @@ updates:
     schedule:
       interval: "daily"
     open-pull-requests-limit: 1
+  - package-ecosystem: "docker"
+    directory: "/installer/cli/deploy/standalone/prometheus"
+    labels:
+      - "dependencies"
+      - "installer"
+    schedule:
+      interval: "daily"
+    open-pull-requests-limit: 1
+  - package-ecosystem: "docker"
+    directory: "/installer/cli/deploy/standalone/prometheus-alertmanager"
+    labels:
+      - "dependencies"
+      - "installer"
+    schedule:
+      interval: "daily"
+    open-pull-requests-limit: 1
   - package-ecosystem: "github-actions"
     directory: "/"
     labels:
@@ -61,23 +75,23 @@ updates:
       interval: "daily"
     ignore:
       - dependency-name: "*"
-        update-types: ["version-update:semver-patch"]
+        update-types: [ "version-update:semver-patch" ]
     open-pull-requests-limit: 1
   - package-ecosystem: "npm"
     directory: "/ui/projects/streampipes/platform-services"
     schedule:
       interval: "daily"
     ignore:
       - dependency-name: "*"
-        update-types: ["version-update:semver-patch"]
+        update-types: [ "version-update:semver-patch" ]
     open-pull-requests-limit: 1
   - package-ecosystem: "npm"
     directory: "/ui/projects/streampipes/shared-ui"
     schedule:
       interval: "daily"
     ignore:
       - dependency-name: "*"
-        update-types: ["version-update:semver-patch"]
+        update-types: [ "version-update:semver-patch" ]
     open-pull-requests-limit: 1
   - package-ecosystem: "pip"
     directory: "/streampipes-client-python"

installer/cli/.env

@@ -39,3 +39,8 @@ SP_PRIORITIZED_PROTOCOL=kafka
 # For database migration in v0.91.0 - set init mode to 'upgrade' to migrate an existing installation
 SP_INFLUX_INIT_MODE=setup
 #SP_INFLUX_INIT_MODE=upgrade
+
+# Monitoring
+# set to true if you want StreamPipes to expose Prometheus metrics
+#SP_SETUP_PROMETHEUS_ENDPOINT=true
+SP_SETUP_PROMETHEUS_ENDPOINT=false

installer/cli/deploy/standalone/backend/docker-compose.yml

@@ -25,6 +25,7 @@ services:
     environment:
       - SP_PRIORITIZED_PROTOCOL=${SP_PRIORITIZED_PROTOCOL:-kafka}
       - SP_MQTT_HOST=${SP_MQTT_HOST:-activemq}
+      - SP_SETUP_PROMETHEUS_ENDPOINT=${SP_SETUP_PROMETHEUS_ENDPOINT}
     logging:
       driver: "json-file"
       options:

installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml

@@ -19,6 +19,8 @@ services:
     image: ${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION}
     depends_on:
       - couchdb
+    environment:
+      - SP_SETUP_PROMETHEUS_ENDPOINT=${SP_SETUP_PROMETHEUS_ENDPOINT}
     volumes:
       - files:/spImages
     logging:

installer/cli/deploy/standalone/prometheus-alertmanager/alertmanager.yml

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+global:
+
+## Add your configuration here

installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.dev.yml

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+  prometheus-alertmanager:
+    ports:
+      - "9093:9093"
+    command:
+      - --config.file=/etc/prometheus-alertmanager/alertmanager.yml
+      - --log.level=debug

installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.yml

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+  prometheus-alertmanager:
+    image: prom/alertmanager:v0.26.0
+    networks:
+      spnet:
+    volumes:
+      - ../prometheus-alertmanager/alertmanager.yml:/etc/prometheus-alertmanager/alertmanager.yml
+    command:
+      - --config.file=/etc/prometheus-alertmanager/alertmanager.yml
+
+networks:
+  spnet:
+    external: true

installer/cli/deploy/standalone/prometheus/docker-compose.dev.yml

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+  prometheus:
+    ports:
+      - "9090:9090"
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --log.level=debug

installer/cli/deploy/standalone/prometheus/docker-compose.yml

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+  prometheus:
+    image: prom/prometheus:v2.48.0
+    networks:
+      spnet:
+    volumes:
+      - ../prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ../prometheus/rules:/etc/prometheus/rules
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+
+networks:
+  spnet:
+    external: true

installer/cli/deploy/standalone/prometheus/prometheus.yml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+global:
+  scrape_interval: 10s # Set the scrape interval to every 10 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+           - prometheus-alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  # add your rules files here
+  # - ./rules/demo.yml
+
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: [ "localhost:9090" ]
+
+  - job_name: "backend"
+    metrics_path: "/streampipes-backend/actuator/prometheus"
+    static_configs:
+      - targets: [ "localhost:8030" ]
+    #bearer_token: ""
+    #bearer_token_file: ""
+
+  - job_name: "extensions-all-iiot"
+    metrics_path: "/actuator/prometheus"
+    static_configs:
+      - targets: [ "localhost:8090" ]

installer/cli/environments/full-monitoring

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[environment:full-monitoring]
+ui
+backend
+extensions-all-jvm
+couchdb
+jobmanager
+taskmanager
+zookeeper
+kafka
+influxdb
+pipeline-elements-all-flink
+grafana
+prometheus
+prometheus-alertmanager

installer/cli/environments/lite-monitoring

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[environment:lite-monitoring]
+ui
+backend
+consul
+couchdb
+kafka
+zookeeper
+influxdb
+extensions-all-jvm
+grafana
+prometheus
+prometheus-alertmanager