FMO-2: Log files to the Message VM

- Add monitoring service - Collect journalctl logs and pass them to messageVM Signed-off-by: Ivan Kuznetsov <[email protected]>
tiiuae · Dec 5, 2024 · ec3129b · ec3129b
1 parent 0b4fd02
commit ec3129b
Show file tree

Hide file tree

Showing 4 changed files with 118 additions and 0 deletions.
diff --git a/hardware/fmo-os-rugged-laptop-7330.nix b/hardware/fmo-os-rugged-laptop-7330.nix
@@ -373,6 +373,15 @@
           };# microvm
           fileSystems."/run/ssh-public-key".options = ["ro"];
           services = {
+            fmo-monitor-service = {
+              enable = true;
+              nats-ip = "192.168.101.111";
+              ca-crt-path = "/var/lib/nats/ca/ca.crt";
+              client-key-path = "/var/lib/nats/certs/client.key";
+              client-crt-path = "/var/lib/nats/certs/client.crt";
+              services = ["fmo-dci.service"];
+              topics = ["dockervm/logs/fmo-dci"];
+            };
             fmo-hostname-service = {
               enable = true;
               hostname-path = "/var/lib/fogdata/hostname";

diff --git a/hardware/fmo-os-rugged-tablet-7230.nix b/hardware/fmo-os-rugged-tablet-7230.nix
@@ -353,6 +353,15 @@
           };# microvm
           fileSystems."/run/ssh-public-key".options = ["ro"];
           services = {
+            fmo-monitor-service = {
+              enable = true;
+              nats-ip = "192.168.101.111";
+              ca-crt-path = "/var/lib/nats/ca/ca.crt";
+              client-key-path = "/var/lib/nats/certs/client.key";
+              client-crt-path = "/var/lib/nats/certs/client.crt";
+              services = ["fmo-dci.service"];
+              topics = ["dockervm/logs/fmo-dci"];
+            };
             fmo-hostname-service = {
               enable = true;
               hostname-path = "/var/lib/fogdata/hostname";

diff --git a/modules/fmo-module-list.nix b/modules/fmo-module-list.nix
@@ -24,4 +24,5 @@
   ./dynamic-device-passthrough-services
   ./dynamic-device-passthrough-services-host
   ./fmo-certs-distribution-host
+  ./fmo-monitoring
 ]
diff --git a/modules/fmo-monitoring/default.nix b/modules/fmo-monitoring/default.nix
@@ -0,0 +1,99 @@
+# Copyright 2022-2024 TII (SSRC) and the Ghaf contributors
+# SPDX-License-Identifier: Apache-2.0
+{ lib, pkgs, config, ... }:
+with lib;
+let
+  cfg = config.services.fmo-monitor-service;
+
+  genServiceTopic = service: topic: '' ["${service}"]="${topic}" '';
+in {
+  options.services.fmo-monitor-service = {
+    enable = mkEnableOption "fmo-monitor-service";
+
+    nats-ip = mkOption {
+      type = types.str;
+      description = "NATS ip address";
+      default = "";
+    };
+
+    nats-port = mkOption {
+      type = types.str;
+      description = "NATS ip address";
+      default = "4222";
+    };
+
+    ca-crt-path = mkOption {
+      type = types.str;
+      description = "Path to CA cert.crt";
+      default = "";
+    };
+
+   client-key-path = mkOption {
+      type = types.str;
+      description = "Paths to client's cert.key";
+      default = "";
+    };
+
+   client-crt-path = mkOption {
+      type = types.str;
+      description = "Paths to client's cert.crt";
+      default = "";
+    };
+
+    services = mkOption {
+      type = types.listOf types.str;
+      description = "Services list to monitor";
+      default = "";
+    };
+
+    topics = mkOption {
+      type = types.listOf types.str;
+      description = "Topics to push messages";
+      default = "";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services."monitor-services" = let
+      monitorScript = pkgs.writeShellScriptBin "monitor-services" ''
+        set -xeuo pipefail
+
+        NATS_URL="nats://${cfg.nats-ip}:${cfg.nats-port}"
+
+        declare -A SERVICES
+        SERVICES=(
+          ${ concatStringsSep "\n" (zipListsWith genServiceTopic cfg.services cfg.topics ) }
+        )
+
+        for SERVICE in "''${!SERVICES[@]}"; do
+          # Get the corresponding NATS topic
+          NATS_TOPIC=''${SERVICES[$SERVICE]}
+
+          # Read logs for the service for the last 5 seconds
+          LOGS=$(journalctl -u "$SERVICE" --since "5 seconds ago" -o cat)
+
+          # If there are logs, send them to NATS
+          if [[ ! -z "$LOGS" ]]; then
+            echo "$LOGS" | while IFS= read -r line; do
+              # Send each log line to the NATS server
+              ${pkgs.natscli}/bin/nats -s "$NATS_URL" --tlsca ${cfg.ca-crt-path} --tlskey ${cfg.client-key-path} --tlscert ${cfg.client-crt-path} pub "$NATS_TOPIC" "$line"
+            done
+          fi
+        done
+      '';
+    in {
+      enable = true;
+      description = "Generate encryption certs";
+      path = [monitorScript];
+      wantedBy = cfg.services;
+      serviceConfig = {
+        RemainAfterExit = false;
+        StandardOutput = "journal";
+        StandardError = "journal";
+        Restart = "always";
+        RestartSec = 5;
+        ExecStart = "${monitorScript}/bin/monitor-services";
+      };
+    };
+  };
+}