Secure scrapes for online instances

This does share some implicit configuration from the target_all but as
we always deploy those together this should be fine.

For online instances which are directly connected to the internet we
didn't secure all metrics. This is now done for:
node (builtin)
mysql (builtin)
fpm (builtin)
nginx (Custom NGINX config)

provision-contest/ansible/roles/grafana/templates/prometheus.yml.j2

@@ -5,11 +5,17 @@ global:
 scrape_configs:
   - job_name: 'grafana'
     static_configs:
-      - targets: ['localhost:{{ grafana_port }}']
+      - targets: ['localhost:3000']
   - job_name: 'prometheus'
     static_configs:
       - targets: ['localhost:9090']
   - job_name: 'db'
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["domserver"] %}
@@ -137,6 +143,12 @@ scrape_configs:
         - {{ hostvars[host].ansible_host }}:9113
 {% endfor %}
   - job_name: 'web_fpm_domserver'
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["domserver"] %}

provision-contest/ansible/roles/prometheus_target_all/files/.gitignore

@@ -0,0 +1,2 @@
+*.key
+*.crt

provision-contest/ansible/roles/prometheus_target_all/tasks/main.yml

@@ -28,9 +28,18 @@
 
 - name: Get HTPassword
   delegate_to: localhost
+  become: false
   shell: "echo {{ PROMETHEUS_PASS }} | htpasswd -inBC 10 \"\" | tr -d ':\n'"
   register: htpassd_shell
 
+- name: Store HTPassword for nginx wrapper
+  copy:
+    content: "prometheus:{{ htpassd_shell.stdout }}"
+    dest: /etc/prometheus/.htpasswd
+    owner: root
+    group: root
+    mode: 0644
+
 - name: Set certificate to encrypt node_exporter traffic
   template:
     owner: prometheus

provision-contest/ansible/roles/prometheus_target_all/templates/web.yml.j2

@@ -0,0 +1,5 @@
+basic_auth_users:
+    prometheus: {{ htpassd_shell.stdout }}
+tls_server_config:
+    cert_file: /etc/prometheus/node_exporter.crt
+    key_file: /etc/prometheus/node_exporter.key

provision-contest/ansible/roles/prometheus_target_web/files/php-fpm-exporter.service

@@ -4,7 +4,7 @@ Documentation=https://github.com/hipages/php-fpm_exporter
 [Service]
 User=www-data
 Restart=always
-ExecStart=/usr/bin/php-fpm_exporter server --phpfpm.fix-process-count --phpfpm.scrape-uri unix:///var/run/php-fpm-domjudge.sock;/fpm_status
+ExecStart=/usr/bin/php-fpm_exporter server --web.listen-address :19253 --phpfpm.fix-process-count --phpfpm.scrape-uri unix:///var/run/php-fpm-domjudge.sock;/fpm_status
 ExecReload=/bin/kill -HUP $MAINPID
 TimeoutStopSec=20s
 SendSIGKILL=no

provision-contest/ansible/roles/prometheus_target_web/tasks/main.yml

@@ -18,11 +18,11 @@
 
 - name: Scrape mysql exporter with TLS encryption
   lineinfile:
-     dest: /etc/default/prometheus-mysqld-exporter
+    dest: /etc/default/prometheus-mysqld-exporter
     state: present
     regexp: '^ARGS=""'
-    line: 'ARGS="--web.config /etc/prometheus/prometheus-authentication.yml"'
-   notify: Restart mysqld-exporter
+    line: 'ARGS="--web.config.file /etc/prometheus/prometheus-authentication.yml"'
+  notify: Restart mysqld-exporter
 
 # Gather PHP-FPM statistics
 # The exporter from this is currently not in deb sources
@@ -66,9 +66,10 @@
 # Gather NGINX statistics,
 # Observe that we use the observed process itself in the monitoring
 - name: Get NGINX status
-  synchronize:
-    src: nginx-status.conf
+  template:
+    src: nginx-status.conf.j2
     dest: /etc/nginx/sites-enabled/nginx-status.conf
+    mode: 0644
   notify: Restart nginx
 
 # In the future add: --web.config /etc/prometheus/prometheus-authentication.yml"'
@@ -79,7 +80,7 @@
     dest: /etc/default/prometheus-nginx-exporter
     state: present
     regexp: '^ARGS=""'
-    line: 'ARGS="-nginx.scrape-uri=http://localhost:8787/basic_status"'
+    line: 'ARGS="-web.listen-address=127.0.0.1:19113 -nginx.scrape-uri=http://localhost:8787/basic_status"'
   notify: Restart nginx-exporter
 
 - name: Create storage dir for exporter settings

provision-contest/ansible/roles/prometheus_target_web/templates/nginx-status.conf.j2

@@ -0,0 +1,34 @@
+server {
+       listen 127.0.0.1:8787;
+       server_name _default_;
+
+       location = /basic_status {
+               stub_status;
+       }
+}
+
+server {
+       listen 0.0.0.0:9113 ssl;
+       ssl_certificate /etc/prometheus/node_exporter.crt;
+       ssl_certificate_key /etc/prometheus/node_exporter.key;
+       ssl_protocols TLSv1.3;
+
+       auth_basic "Prometheus scraping";
+       auth_basic_user_file /etc/prometheus/.htpasswd;
+       location / {
+               proxy_pass http://127.0.0.1:19113;
+       }
+}
+
+server {
+       listen 0.0.0.0:9253 ssl;
+       ssl_certificate /etc/prometheus/node_exporter.crt;
+       ssl_certificate_key /etc/prometheus/node_exporter.key;
+       ssl_protocols TLSv1.3;
+
+       auth_basic "Prometheus scraping";
+       auth_basic_user_file /etc/prometheus/.htpasswd;
+       location / {
+               proxy_pass http://127.0.0.1:19253;
+       }
+}