From 40222dd36ade9563d5001ce41d929e072f0ab6f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 31 May 2023 09:00:52 +0200 Subject: [PATCH] Add a collectd module to export the cache build stats to collectd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aurélien Bompard --- .s2i/bin/assemble | 2 - .s2i/run-collectd.sh | 13 +++ changelog.d/PRXXX.feature | 1 + config/collectd-fmn.conf.example | 24 ++++ config/collectd-types.db | 5 + devel/ansible/playbook.yml | 1 + .../ansible/roles/collectd/handlers/main.yml | 8 ++ devel/ansible/roles/collectd/tasks/main.yml | 52 +++++++++ .../roles/collectd/templates/collectd.conf | 60 ++++++++++ fmn/core/collectd.py | 109 ++++++++++++++++++ 10 files changed, 273 insertions(+), 2 deletions(-) create mode 100755 .s2i/run-collectd.sh create mode 100644 changelog.d/PRXXX.feature create mode 100644 config/collectd-fmn.conf.example create mode 100644 config/collectd-types.db create mode 100644 devel/ansible/roles/collectd/handlers/main.yml create mode 100644 devel/ansible/roles/collectd/tasks/main.yml create mode 100644 devel/ansible/roles/collectd/templates/collectd.conf create mode 100644 fmn/core/collectd.py diff --git a/.s2i/bin/assemble b/.s2i/bin/assemble index 1c643b94e..ec519d68b 100755 --- a/.s2i/bin/assemble +++ b/.s2i/bin/assemble @@ -40,8 +40,6 @@ pip install redis cd frontend npm install -env - if [ "$FEDORA_ENV" == "staging" ]; then npm run build:staging else diff --git a/.s2i/run-collectd.sh b/.s2i/run-collectd.sh new file mode 100755 index 000000000..22ae8220b --- /dev/null +++ b/.s2i/run-collectd.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: Contributors to the Fedora Project +# +# SPDX-License-Identifier: MIT + +set -e + +# We install the app in a specific virtualenv: +export PATH=/opt/app-root/src/.local/venvs/fmn/bin:$PATH + +# Run collectd +collectd -f -C /etc/fmn/collectd.conf diff --git a/changelog.d/PRXXX.feature b/changelog.d/PRXXX.feature new file mode 100644 index 000000000..46a3072f2 --- /dev/null +++ b/changelog.d/PRXXX.feature @@ -0,0 +1 @@ +Send the cache building stats to collectd diff --git a/config/collectd-fmn.conf.example b/config/collectd-fmn.conf.example new file mode 100644 index 000000000..41ef76eca --- /dev/null +++ b/config/collectd-fmn.conf.example @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Contributors to the Fedora Project +# +# SPDX-License-Identifier: MIT + +TypesDB "/usr/share/collectd/fmn-types.db" + + + Globals true + + + + LogTraces true + Interactive false + # ModulePath "/opt/app-root/src" + Import "fmn.core.collectd" + + + ## Interval between two collections. The collectd default of 10 seconds is + ## way too short, this plugin sets the default to 1h (3600s). Adjust + ## depending on how frequently the cache is rebuilt. Remember that if you + ## change the interval, you'll have to recreate your RRD files. + # Interval 3600 + + diff --git a/config/collectd-types.db b/config/collectd-types.db new file mode 100644 index 000000000..788344e0e --- /dev/null +++ b/config/collectd-types.db @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Contributors to the Fedora Project +# +# SPDX-License-Identifier: MIT + +fmn_cache value:GAUGE:0:U diff --git a/devel/ansible/playbook.yml b/devel/ansible/playbook.yml index 5236cfa99..81ab73f8c 100644 --- a/devel/ansible/playbook.yml +++ b/devel/ansible/playbook.yml @@ -23,3 +23,4 @@ - consumer - sender - redis + - collectd diff --git a/devel/ansible/roles/collectd/handlers/main.yml b/devel/ansible/roles/collectd/handlers/main.yml new file mode 100644 index 000000000..8704b32ef --- /dev/null +++ b/devel/ansible/roles/collectd/handlers/main.yml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Contributors to the Fedora Project +# +# SPDX-License-Identifier: MIT + +- name: restart collectd + service: + name: collectd + state: restarted diff --git a/devel/ansible/roles/collectd/tasks/main.yml b/devel/ansible/roles/collectd/tasks/main.yml new file mode 100644 index 000000000..73a350597 --- /dev/null +++ b/devel/ansible/roles/collectd/tasks/main.yml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Contributors to the Fedora Project +# +# SPDX-License-Identifier: MIT + +- name: install packages + dnf: + name: + - collectd + - collectd-python + - collectd-rrdtool + - collectd-write_syslog + +- name: allow collectd to do network connections + seboolean: + name: collectd_tcp_network_connect + persistent: true + state: true + +- name: get FMN's virtualenv + shell: + cmd: "poetry run python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())'" + chdir: /home/vagrant/fmn/ + register: fmn_venv_lib + changed_when: false + +- name: get FMN's virtualenv + shell: + cmd: "poetry run python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib(True))'" + chdir: /home/vagrant/fmn/ + register: fmn_venv_arch + changed_when: false + +- name: copy the collectd config file over + template: + src: collectd.conf + dest: /etc/collectd.conf + notify: + - restart collectd + +- name: copy the collectd types DB + copy: + remote_src: true + src: /home/vagrant/fmn/config/collectd-types.db + dest: /usr/share/collectd/fmn-types.db + notify: + - restart collectd + +- name: set the collectd service to start + service: + name: collectd + enabled: true + state: started diff --git a/devel/ansible/roles/collectd/templates/collectd.conf b/devel/ansible/roles/collectd/templates/collectd.conf new file mode 100644 index 000000000..e9cbfce80 --- /dev/null +++ b/devel/ansible/roles/collectd/templates/collectd.conf @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Contributors to the Fedora Project +# +# SPDX-License-Identifier: MIT + +Hostname "{{ ansible_hostname }}" +FQDNLookup true +#BaseDir "/usr/var/lib/collectd" +#PIDFile "/usr/var/run/collectd.pid" +#PluginDir "/usr/lib/collectd" +#Interval 10 +#ReadThreads 5 + +# This is the default but it needs to be defined so we can add more DB files later. +TypesDB "/usr/share/collectd/types.db" + +LoadPlugin logfile + + + LogLevel "info" + File STDOUT + Timestamp true + + +# Write data to disk +LoadPlugin csv + + DataDir "/var/lib/collectd/csv" + StoreRates true + +LoadPlugin rrdtool + + DataDir "/var/lib/collectd/rrd" + CacheFlush 120 + WritesPerSecond 50 + + +# FMN + +TypesDB "/usr/share/collectd/fmn-types.db" + + + Globals true + + + + LogTraces true + Interactive false + ModulePath "{{ fmn_venv_lib.stdout }}" + ModulePath "{{ fmn_venv_arch.stdout }}" + ModulePath "/home/vagrant/fmn" + Import "fmn.core.collectd" + + + ## Interval between two collections. The collectd default of 10 seconds is + ## way too short, this plugin sets the default to 1h (3600s). Adjust + ## depending on how frequently the cache is rebuilt. Remember that if you + ## change the interval, you'll have to recreate your RRD files. + # Interval 3600 + + diff --git a/fmn/core/collectd.py b/fmn/core/collectd.py new file mode 100644 index 000000000..26a5141c6 --- /dev/null +++ b/fmn/core/collectd.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Contributors to the Fedora Project +# +# SPDX-License-Identifier: MIT + +import asyncio +import os +from datetime import datetime +from functools import partial + +import collectd +from cashews import cache + +from fmn.cache import configure_cache + +CONFIG = { + "Interval": "3600", + "Hostname": None, +} + + +class Collector: + def __init__(self, config): + self.config = config + self._loop = None + + def setup(self): + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + configure_cache() + + def shutdown(self): + self._loop.run_until_complete(cache.close()) + to_cancel = asyncio.all_tasks(self._loop) + for task in to_cancel: + task.cancel() + self._loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True)) + self._loop.run_until_complete(self._loop.shutdown_asyncgens()) + self._loop.run_until_complete(self._loop.shutdown_default_executor()) + asyncio.set_event_loop(None) + self._loop.close() + + def collect(self): + self._loop.run_until_complete(self._collect()) + + async def _collect(self): + now = datetime.now().timestamp() + async for key in cache.scan("duration:*"): + duration = await cache.get(key) + if duration is None: + continue + _, name, when = key.split(":", 2) + when = datetime.fromisoformat(when).timestamp() + if now - when > int(self.config["Interval"]) * 24: + collectd.debug(f"Not dispatching {name} at {when}: too old") + continue + collectd.debug(f"Dispatching {name} at {when}: {duration!r}") + collectd.info(f"Dispatching {name} at {when}: {duration!r}") + self._loop.run_in_executor( + None, + partial( + self._dispatch, + duration, + "cache", + timestamp=when, + subname=name, + category="cache", + ), + ) + + def _dispatch(self, value, name, timestamp=None, subname=None, category=None): + vl = collectd.Values() + vl.type = f"fmn_{name}" + vl.plugin = category or name + host = self.config.get("Hostname") + if host: + vl.host = host + vl.interval = int(self.config["Interval"]) + if subname is not None: + vl.type_instance = subname + if not hasattr(value, "__iter__"): + value = [value] + vl.dispatch(values=value) + + +def configure(plugin_config): + config = CONFIG.copy() + for conf_entry in plugin_config.children: + collectd.debug(f"{conf_entry.key} = {conf_entry.values}") + try: + if conf_entry.key == "SetEnv": + envvar, value = conf_entry.values + os.environ[envvar] = value + else: + if len(conf_entry.values) != 1: + raise ValueError + config[conf_entry.key] = conf_entry.values[0] + except ValueError: + collectd.warning( + f"Invalid configuration value for {conf_entry.key}: {conf_entry.values!r}" + ) + continue + + collector = Collector(config=config) + collectd.register_init(collector.setup) + collectd.register_shutdown(collector.shutdown) + collectd.register_read(collector.collect, int(config["Interval"])) + + +collectd.register_config(configure)