From 06016fc0f3d09a656799e3e3d8f666df58ad5b32 Mon Sep 17 00:00:00 2001 From: Tim <50115603+bossenti@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:17:48 +0100 Subject: [PATCH] extend CLI by monitoring stack (#2256) * chore: extend CLI by monitoring stack * chore: extend dependabot config by new docker images * chore: add env variable to backend & extensions service container --- .github/dependabot.yml | 24 ++++++++-- installer/cli/.env | 5 ++ .../standalone/backend/docker-compose.yml | 1 + .../extensions-all-jvm/docker-compose.yml | 2 + .../prometheus-alertmanager/alertmanager.yml | 18 +++++++ .../docker-compose.dev.yml | 23 +++++++++ .../docker-compose.yml | 29 ++++++++++++ .../prometheus/docker-compose.dev.yml | 23 +++++++++ .../standalone/prometheus/docker-compose.yml | 30 ++++++++++++ .../standalone/prometheus/prometheus.yml | 47 +++++++++++++++++++ installer/cli/environments/full-monitoring | 29 ++++++++++++ installer/cli/environments/lite-monitoring | 27 +++++++++++ 12 files changed, 253 insertions(+), 5 deletions(-) create mode 100644 installer/cli/deploy/standalone/prometheus-alertmanager/alertmanager.yml create mode 100644 installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.dev.yml create mode 100644 installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.yml create mode 100644 installer/cli/deploy/standalone/prometheus/docker-compose.dev.yml create mode 100644 installer/cli/deploy/standalone/prometheus/docker-compose.yml create mode 100644 installer/cli/deploy/standalone/prometheus/prometheus.yml create mode 100644 installer/cli/environments/full-monitoring create mode 100644 installer/cli/environments/lite-monitoring diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 822b365e9b..d48dddaa50 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -22,7 +22,6 @@ updates: schedule: interval: "daily" open-pull-requests-limit: 1 - - package-ecosystem: "docker" directory: "/installer/compose" labels: @@ -31,7 +30,6 @@ updates: schedule: interval: "daily" open-pull-requests-limit: 1 - - package-ecosystem: "docker" directory: "/installer/cli/deploy/standalone/grafana" labels: @@ -40,6 +38,22 @@ updates: schedule: interval: "daily" open-pull-requests-limit: 1 + - package-ecosystem: "docker" + directory: "/installer/cli/deploy/standalone/prometheus" + labels: + - "dependencies" + - "installer" + schedule: + interval: "daily" + open-pull-requests-limit: 1 + - package-ecosystem: "docker" + directory: "/installer/cli/deploy/standalone/prometheus-alertmanager" + labels: + - "dependencies" + - "installer" + schedule: + interval: "daily" + open-pull-requests-limit: 1 - package-ecosystem: "github-actions" directory: "/" labels: @@ -61,7 +75,7 @@ updates: interval: "daily" ignore: - dependency-name: "*" - update-types: ["version-update:semver-patch"] + update-types: [ "version-update:semver-patch" ] open-pull-requests-limit: 1 - package-ecosystem: "npm" directory: "/ui/projects/streampipes/platform-services" @@ -69,7 +83,7 @@ updates: interval: "daily" ignore: - dependency-name: "*" - update-types: ["version-update:semver-patch"] + update-types: [ "version-update:semver-patch" ] open-pull-requests-limit: 1 - package-ecosystem: "npm" directory: "/ui/projects/streampipes/shared-ui" @@ -77,7 +91,7 @@ updates: interval: "daily" ignore: - dependency-name: "*" - update-types: ["version-update:semver-patch"] + update-types: [ "version-update:semver-patch" ] open-pull-requests-limit: 1 - package-ecosystem: "pip" directory: "/streampipes-client-python" diff --git a/installer/cli/.env b/installer/cli/.env index 4f899ce21f..ce976949ee 100644 --- a/installer/cli/.env +++ b/installer/cli/.env @@ -39,3 +39,8 @@ SP_PRIORITIZED_PROTOCOL=kafka # For database migration in v0.91.0 - set init mode to 'upgrade' to migrate an existing installation SP_INFLUX_INIT_MODE=setup #SP_INFLUX_INIT_MODE=upgrade + +# Monitoring +# set to true if you want StreamPipes to expose Prometheus metrics +#SP_SETUP_PROMETHEUS_ENDPOINT=true +SP_SETUP_PROMETHEUS_ENDPOINT=false diff --git a/installer/cli/deploy/standalone/backend/docker-compose.yml b/installer/cli/deploy/standalone/backend/docker-compose.yml index c6f3200222..9264132f64 100644 --- a/installer/cli/deploy/standalone/backend/docker-compose.yml +++ b/installer/cli/deploy/standalone/backend/docker-compose.yml @@ -25,6 +25,7 @@ services: environment: - SP_PRIORITIZED_PROTOCOL=${SP_PRIORITIZED_PROTOCOL:-kafka} - SP_MQTT_HOST=${SP_MQTT_HOST:-activemq} + - SP_SETUP_PROMETHEUS_ENDPOINT=${SP_SETUP_PROMETHEUS_ENDPOINT} logging: driver: "json-file" options: diff --git a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml b/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml index a446551390..90686788d6 100644 --- a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml +++ b/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml @@ -19,6 +19,8 @@ services: image: ${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION} depends_on: - couchdb + environment: + - SP_SETUP_PROMETHEUS_ENDPOINT=${SP_SETUP_PROMETHEUS_ENDPOINT} volumes: - files:/spImages logging: diff --git a/installer/cli/deploy/standalone/prometheus-alertmanager/alertmanager.yml b/installer/cli/deploy/standalone/prometheus-alertmanager/alertmanager.yml new file mode 100644 index 0000000000..bbc7ee7fdf --- /dev/null +++ b/installer/cli/deploy/standalone/prometheus-alertmanager/alertmanager.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +global: + +## Add your configuration here diff --git a/installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.dev.yml b/installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.dev.yml new file mode 100644 index 0000000000..c6e6c834a0 --- /dev/null +++ b/installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.dev.yml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + prometheus-alertmanager: + ports: + - "9093:9093" + command: + - --config.file=/etc/prometheus-alertmanager/alertmanager.yml + - --log.level=debug diff --git a/installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.yml b/installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.yml new file mode 100644 index 0000000000..4f7ed5316b --- /dev/null +++ b/installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.yml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + prometheus-alertmanager: + image: prom/alertmanager:v0.26.0 + networks: + spnet: + volumes: + - ../prometheus-alertmanager/alertmanager.yml:/etc/prometheus-alertmanager/alertmanager.yml + command: + - --config.file=/etc/prometheus-alertmanager/alertmanager.yml + +networks: + spnet: + external: true \ No newline at end of file diff --git a/installer/cli/deploy/standalone/prometheus/docker-compose.dev.yml b/installer/cli/deploy/standalone/prometheus/docker-compose.dev.yml new file mode 100644 index 0000000000..d5728b61da --- /dev/null +++ b/installer/cli/deploy/standalone/prometheus/docker-compose.dev.yml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + prometheus: + ports: + - "9090:9090" + command: + - --config.file=/etc/prometheus/prometheus.yml + - --log.level=debug diff --git a/installer/cli/deploy/standalone/prometheus/docker-compose.yml b/installer/cli/deploy/standalone/prometheus/docker-compose.yml new file mode 100644 index 0000000000..15bcfd0caa --- /dev/null +++ b/installer/cli/deploy/standalone/prometheus/docker-compose.yml @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + prometheus: + image: prom/prometheus:v2.48.0 + networks: + spnet: + volumes: + - ../prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - ../prometheus/rules:/etc/prometheus/rules + command: + - --config.file=/etc/prometheus/prometheus.yml + +networks: + spnet: + external: true \ No newline at end of file diff --git a/installer/cli/deploy/standalone/prometheus/prometheus.yml b/installer/cli/deploy/standalone/prometheus/prometheus.yml new file mode 100644 index 0000000000..daf866fccb --- /dev/null +++ b/installer/cli/deploy/standalone/prometheus/prometheus.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +global: + scrape_interval: 10s # Set the scrape interval to every 10 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +alerting: + alertmanagers: + - static_configs: + - targets: + - prometheus-alertmanager:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + # add your rules files here + # - ./rules/demo.yml + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: [ "localhost:9090" ] + + - job_name: "backend" + metrics_path: "/streampipes-backend/actuator/prometheus" + static_configs: + - targets: [ "localhost:8030" ] + #bearer_token: "" + #bearer_token_file: "" + + - job_name: "extensions-all-iiot" + metrics_path: "/actuator/prometheus" + static_configs: + - targets: [ "localhost:8090" ] diff --git a/installer/cli/environments/full-monitoring b/installer/cli/environments/full-monitoring new file mode 100644 index 0000000000..3cc41edb32 --- /dev/null +++ b/installer/cli/environments/full-monitoring @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[environment:full-monitoring] +ui +backend +extensions-all-jvm +couchdb +jobmanager +taskmanager +zookeeper +kafka +influxdb +pipeline-elements-all-flink +grafana +prometheus +prometheus-alertmanager diff --git a/installer/cli/environments/lite-monitoring b/installer/cli/environments/lite-monitoring new file mode 100644 index 0000000000..d99f12c3e4 --- /dev/null +++ b/installer/cli/environments/lite-monitoring @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[environment:lite-monitoring] +ui +backend +consul +couchdb +kafka +zookeeper +influxdb +extensions-all-jvm +grafana +prometheus +prometheus-alertmanager