Add a collectd module to export the cache build stats to collectd

Signed-off-by: Aurélien Bompard <[email protected]>

.s2i/bin/assemble

@@ -40,8 +40,6 @@ pip install redis
 cd frontend
 npm install
 
-env
-
 if [ "$FEDORA_ENV" == "staging" ]; then
     npm run build:staging
 else

.s2i/run-collectd.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: Contributors to the Fedora Project
+#
+# SPDX-License-Identifier: MIT
+
+set -e
+
+# We install the app in a specific virtualenv:
+export PATH=/opt/app-root/src/.local/venvs/fmn/bin:$PATH
+
+# Run collectd
+collectd -f -C /etc/fmn/collectd.conf

changelog.d/PRXXX.feature

@@ -0,0 +1 @@
+Send the cache building stats to collectd

config/collectd-fmn.conf.example

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: Contributors to the Fedora Project
+#
+# SPDX-License-Identifier: MIT
+
+TypesDB "/usr/share/collectd/fmn-types.db"
+
+<LoadPlugin python>
+  Globals true
+</LoadPlugin>
+
+<Plugin python>
+  LogTraces true
+  Interactive false
+  # ModulePath "/opt/app-root/src"
+  Import "fmn.core.collectd"
+
+  <Module "fmn.core.collectd">
+    ## Interval between two collections. The collectd default of 10 seconds is
+    ## way too short, this plugin sets the default to 1h (3600s). Adjust
+    ## depending on how frequently the cache is rebuilt. Remember that if you
+    ## change the interval, you'll have to recreate your RRD files.
+    # Interval 3600
+  </Module>
+</Plugin>

config/collectd-types.db

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: Contributors to the Fedora Project
+#
+# SPDX-License-Identifier: MIT
+
+fmn_cache        value:GAUGE:0:U

devel/ansible/playbook.yml

@@ -23,3 +23,4 @@
     - consumer
     - sender
     - redis
+    - collectd

devel/ansible/roles/collectd/handlers/main.yml

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: Contributors to the Fedora Project
+#
+# SPDX-License-Identifier: MIT
+
+- name: restart collectd
+  service:
+    name: collectd
+    state: restarted

devel/ansible/roles/collectd/tasks/main.yml

@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: Contributors to the Fedora Project
+#
+# SPDX-License-Identifier: MIT
+
+- name: install packages
+  dnf:
+    name:
+      - collectd
+      - collectd-python
+      - collectd-rrdtool
+      - collectd-write_syslog
+
+- name: allow collectd to do network connections
+  seboolean:
+    name: collectd_tcp_network_connect
+    persistent: true
+    state: true
+
+- name: get FMN's virtualenv
+  shell:
+    cmd: "poetry run python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())'"
+    chdir: /home/vagrant/fmn/
+  register: fmn_venv_lib
+  changed_when: false
+
+- name: get FMN's virtualenv
+  shell:
+    cmd: "poetry run python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib(True))'"
+    chdir: /home/vagrant/fmn/
+  register: fmn_venv_arch
+  changed_when: false
+
+- name: copy the collectd config file over
+  template:
+    src: collectd.conf
+    dest: /etc/collectd.conf
+  notify:
+    - restart collectd
+
+- name: copy the collectd types DB
+  copy:
+    remote_src: true
+    src: /home/vagrant/fmn/config/collectd-types.db
+    dest: /usr/share/collectd/fmn-types.db
+  notify:
+    - restart collectd
+
+- name: set the collectd service to start
+  service:
+    name: collectd
+    enabled: true
+    state: started

devel/ansible/roles/collectd/templates/collectd.conf

@@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: Contributors to the Fedora Project
+#
+# SPDX-License-Identifier: MIT
+
+Hostname    "{{ ansible_hostname }}"
+FQDNLookup   true
+#BaseDir     "/usr/var/lib/collectd"
+#PIDFile     "/usr/var/run/collectd.pid"
+#PluginDir   "/usr/lib/collectd"
+#Interval     10
+#ReadThreads  5
+
+# This is the default but it needs to be defined so we can add more DB files later.
+TypesDB     "/usr/share/collectd/types.db"
+
+LoadPlugin logfile
+
+<Plugin LogFile>
+    LogLevel "info"
+    File STDOUT
+    Timestamp true
+</Plugin>
+
+# Write data to disk
+LoadPlugin csv
+<Plugin "csv">
+  DataDir "/var/lib/collectd/csv"
+  StoreRates true
+</Plugin>
+LoadPlugin rrdtool
+<Plugin "rrdtool">
+  DataDir "/var/lib/collectd/rrd"
+  CacheFlush 120
+  WritesPerSecond 50
+</Plugin>
+
+# FMN
+
+TypesDB "/usr/share/collectd/fmn-types.db"
+
+<LoadPlugin python>
+  Globals true
+</LoadPlugin>
+
+<Plugin python>
+  LogTraces true
+  Interactive false
+  ModulePath "{{ fmn_venv_lib.stdout }}"
+  ModulePath "{{ fmn_venv_arch.stdout }}"
+  ModulePath "/home/vagrant/fmn"
+  Import "fmn.core.collectd"
+
+  <Module "fmn.core.collectd">
+    ## Interval between two collections. The collectd default of 10 seconds is
+    ## way too short, this plugin sets the default to 1h (3600s). Adjust
+    ## depending on how frequently the cache is rebuilt. Remember that if you
+    ## change the interval, you'll have to recreate your RRD files.
+    # Interval 3600
+  </Module>
+</Plugin>

fmn/core/collectd.py

@@ -0,0 +1,109 @@
+# SPDX-FileCopyrightText: Contributors to the Fedora Project
+#
+# SPDX-License-Identifier: MIT
+
+import asyncio
+import os
+from datetime import datetime
+from functools import partial
+
+import collectd
+from cashews import cache
+
+from fmn.cache import configure_cache
+
+CONFIG = {
+    "Interval": "3600",
+    "Hostname": None,
+}
+
+
+class Collector:
+    def __init__(self, config):
+        self.config = config
+        self._loop = None
+
+    def setup(self):
+        self._loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(self._loop)
+        configure_cache()
+
+    def shutdown(self):
+        self._loop.run_until_complete(cache.close())
+        to_cancel = asyncio.all_tasks(self._loop)
+        for task in to_cancel:
+            task.cancel()
+        self._loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True))
+        self._loop.run_until_complete(self._loop.shutdown_asyncgens())
+        self._loop.run_until_complete(self._loop.shutdown_default_executor())
+        asyncio.set_event_loop(None)
+        self._loop.close()
+
+    def collect(self):
+        self._loop.run_until_complete(self._collect())
+
+    async def _collect(self):
+        now = datetime.now().timestamp()
+        async for key in cache.scan("duration:*"):
+            duration = await cache.get(key)
+            if duration is None:
+                continue
+            _, name, when = key.split(":", 2)
+            when = datetime.fromisoformat(when).timestamp()
+            if now - when > int(self.config["Interval"]) * 24:
+                collectd.debug(f"Not dispatching {name} at {when}: too old")
+                continue
+            collectd.debug(f"Dispatching {name} at {when}: {duration!r}")
+            collectd.info(f"Dispatching {name} at {when}: {duration!r}")
+            self._loop.run_in_executor(
+                None,
+                partial(
+                    self._dispatch,
+                    duration,
+                    "cache",
+                    timestamp=when,
+                    subname=name,
+                    category="cache",
+                ),
+            )
+
+    def _dispatch(self, value, name, timestamp=None, subname=None, category=None):
+        vl = collectd.Values()
+        vl.type = f"fmn_{name}"
+        vl.plugin = category or name
+        host = self.config.get("Hostname")
+        if host:
+            vl.host = host
+        vl.interval = int(self.config["Interval"])
+        if subname is not None:
+            vl.type_instance = subname
+        if not hasattr(value, "__iter__"):
+            value = [value]
+        vl.dispatch(values=value)
+
+
+def configure(plugin_config):
+    config = CONFIG.copy()
+    for conf_entry in plugin_config.children:
+        collectd.debug(f"{conf_entry.key} = {conf_entry.values}")
+        try:
+            if conf_entry.key == "SetEnv":
+                envvar, value = conf_entry.values
+                os.environ[envvar] = value
+            else:
+                if len(conf_entry.values) != 1:
+                    raise ValueError
+                config[conf_entry.key] = conf_entry.values[0]
+        except ValueError:
+            collectd.warning(
+                f"Invalid configuration value for {conf_entry.key}: {conf_entry.values!r}"
+            )
+            continue
+
+    collector = Collector(config=config)
+    collectd.register_init(collector.setup)
+    collectd.register_shutdown(collector.shutdown)
+    collectd.register_read(collector.collect, int(config["Interval"]))
+
+
+collectd.register_config(configure)