From 0d52c5b4678cee6937119a4a0f6bb3ff753cc657 Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Mon, 5 Aug 2024 22:10:18 +0200 Subject: [PATCH] Secure scrapes for online instances This does share some implicit configuration from the target_all but as we always deploy those together this should be fine. For online instances which are directly connected to the internet we didn't secure all metrics. This is now done for: node (builtin) mysql (builtin) fpm (builtin) nginx (Custom NGINX config) --- .../roles/grafana/templates/prometheus.yml.j2 | 14 +++++++- .../prometheus_target_all/files/.gitignore | 2 ++ .../prometheus_target_all/tasks/main.yml | 9 +++++ .../templates/web.yml.j2 | 5 +++ .../files/php-fpm-exporter.service | 2 +- .../prometheus_target_web/tasks/main.yml | 13 +++---- .../templates/nginx-status.conf.j2 | 34 +++++++++++++++++++ 7 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 provision-contest/ansible/roles/prometheus_target_all/files/.gitignore create mode 100644 provision-contest/ansible/roles/prometheus_target_all/templates/web.yml.j2 create mode 100644 provision-contest/ansible/roles/prometheus_target_web/templates/nginx-status.conf.j2 diff --git a/provision-contest/ansible/roles/grafana/templates/prometheus.yml.j2 b/provision-contest/ansible/roles/grafana/templates/prometheus.yml.j2 index d7ba872f..8246bf75 100644 --- a/provision-contest/ansible/roles/grafana/templates/prometheus.yml.j2 +++ b/provision-contest/ansible/roles/grafana/templates/prometheus.yml.j2 @@ -5,11 +5,17 @@ global: scrape_configs: - job_name: 'grafana' static_configs: - - targets: ['localhost:{{ grafana_port }}'] + - targets: ['localhost:3000'] - job_name: 'prometheus' static_configs: - targets: ['localhost:9090'] - job_name: 'db' + basic_auth: + username: "prometheus" + password: "{{ PROMETHEUS_PASS }}" + tls_config: + insecure_skip_verify: true + scheme: https static_configs: - targets: {% for host in groups["domserver"] %} @@ -137,6 +143,12 @@ scrape_configs: - {{ hostvars[host].ansible_host }}:9113 {% endfor %} - job_name: 'web_fpm_domserver' + basic_auth: + username: "prometheus" + password: "{{ PROMETHEUS_PASS }}" + tls_config: + insecure_skip_verify: true + scheme: https static_configs: - targets: {% for host in groups["domserver"] %} diff --git a/provision-contest/ansible/roles/prometheus_target_all/files/.gitignore b/provision-contest/ansible/roles/prometheus_target_all/files/.gitignore new file mode 100644 index 00000000..275ece24 --- /dev/null +++ b/provision-contest/ansible/roles/prometheus_target_all/files/.gitignore @@ -0,0 +1,2 @@ +*.key +*.crt diff --git a/provision-contest/ansible/roles/prometheus_target_all/tasks/main.yml b/provision-contest/ansible/roles/prometheus_target_all/tasks/main.yml index a509ef4c..7b2f81dc 100644 --- a/provision-contest/ansible/roles/prometheus_target_all/tasks/main.yml +++ b/provision-contest/ansible/roles/prometheus_target_all/tasks/main.yml @@ -28,9 +28,18 @@ - name: Get HTPassword delegate_to: localhost + become: false shell: "echo {{ PROMETHEUS_PASS }} | htpasswd -inBC 10 \"\" | tr -d ':\n'" register: htpassd_shell +- name: Store HTPassword for nginx wrapper + copy: + content: "prometheus:{{ htpassd_shell.stdout }}" + dest: /etc/prometheus/.htpasswd + owner: root + group: root + mode: 0644 + - name: Set certificate to encrypt node_exporter traffic template: owner: prometheus diff --git a/provision-contest/ansible/roles/prometheus_target_all/templates/web.yml.j2 b/provision-contest/ansible/roles/prometheus_target_all/templates/web.yml.j2 new file mode 100644 index 00000000..4d1a35ac --- /dev/null +++ b/provision-contest/ansible/roles/prometheus_target_all/templates/web.yml.j2 @@ -0,0 +1,5 @@ +basic_auth_users: + prometheus: {{ htpassd_shell.stdout }} +tls_server_config: + cert_file: /etc/prometheus/node_exporter.crt + key_file: /etc/prometheus/node_exporter.key diff --git a/provision-contest/ansible/roles/prometheus_target_web/files/php-fpm-exporter.service b/provision-contest/ansible/roles/prometheus_target_web/files/php-fpm-exporter.service index 94e5185e..9d6ca255 100644 --- a/provision-contest/ansible/roles/prometheus_target_web/files/php-fpm-exporter.service +++ b/provision-contest/ansible/roles/prometheus_target_web/files/php-fpm-exporter.service @@ -4,7 +4,7 @@ Documentation=https://github.com/hipages/php-fpm_exporter [Service] User=www-data Restart=always -ExecStart=/usr/bin/php-fpm_exporter server --phpfpm.fix-process-count --phpfpm.scrape-uri unix:///var/run/php-fpm-domjudge.sock;/fpm_status +ExecStart=/usr/bin/php-fpm_exporter server --web.listen-address :19253 --phpfpm.fix-process-count --phpfpm.scrape-uri unix:///var/run/php-fpm-domjudge.sock;/fpm_status ExecReload=/bin/kill -HUP $MAINPID TimeoutStopSec=20s SendSIGKILL=no diff --git a/provision-contest/ansible/roles/prometheus_target_web/tasks/main.yml b/provision-contest/ansible/roles/prometheus_target_web/tasks/main.yml index 069520ae..609c7db8 100644 --- a/provision-contest/ansible/roles/prometheus_target_web/tasks/main.yml +++ b/provision-contest/ansible/roles/prometheus_target_web/tasks/main.yml @@ -18,11 +18,11 @@ - name: Scrape mysql exporter with TLS encryption lineinfile: - dest: /etc/default/prometheus-mysqld-exporter + dest: /etc/default/prometheus-mysqld-exporter state: present regexp: '^ARGS=""' - line: 'ARGS="--web.config /etc/prometheus/prometheus-authentication.yml"' - notify: Restart mysqld-exporter + line: 'ARGS="--web.config.file /etc/prometheus/prometheus-authentication.yml"' + notify: Restart mysqld-exporter # Gather PHP-FPM statistics # The exporter from this is currently not in deb sources @@ -66,9 +66,10 @@ # Gather NGINX statistics, # Observe that we use the observed process itself in the monitoring - name: Get NGINX status - synchronize: - src: nginx-status.conf + template: + src: nginx-status.conf.j2 dest: /etc/nginx/sites-enabled/nginx-status.conf + mode: 0644 notify: Restart nginx # In the future add: --web.config /etc/prometheus/prometheus-authentication.yml"' @@ -79,7 +80,7 @@ dest: /etc/default/prometheus-nginx-exporter state: present regexp: '^ARGS=""' - line: 'ARGS="-nginx.scrape-uri=http://localhost:8787/basic_status"' + line: 'ARGS="-web.listen-address=127.0.0.1:19113 -nginx.scrape-uri=http://localhost:8787/basic_status"' notify: Restart nginx-exporter - name: Create storage dir for exporter settings diff --git a/provision-contest/ansible/roles/prometheus_target_web/templates/nginx-status.conf.j2 b/provision-contest/ansible/roles/prometheus_target_web/templates/nginx-status.conf.j2 new file mode 100644 index 00000000..e5397fdb --- /dev/null +++ b/provision-contest/ansible/roles/prometheus_target_web/templates/nginx-status.conf.j2 @@ -0,0 +1,34 @@ +server { + listen 127.0.0.1:8787; + server_name _default_; + + location = /basic_status { + stub_status; + } +} + +server { + listen 0.0.0.0:9113 ssl; + ssl_certificate /etc/prometheus/node_exporter.crt; + ssl_certificate_key /etc/prometheus/node_exporter.key; + ssl_protocols TLSv1.3; + + auth_basic "Prometheus scraping"; + auth_basic_user_file /etc/prometheus/.htpasswd; + location / { + proxy_pass http://127.0.0.1:19113; + } +} + +server { + listen 0.0.0.0:9253 ssl; + ssl_certificate /etc/prometheus/node_exporter.crt; + ssl_certificate_key /etc/prometheus/node_exporter.key; + ssl_protocols TLSv1.3; + + auth_basic "Prometheus scraping"; + auth_basic_user_file /etc/prometheus/.htpasswd; + location / { + proxy_pass http://127.0.0.1:19253; + } +}