From eea21ab9a0a42f1b5bca453af4c819ad7ada0003 Mon Sep 17 00:00:00 2001 From: Manuel Bluhm Date: Wed, 20 Nov 2024 22:25:49 +0400 Subject: [PATCH] Add multiple addresses to admin nixos module Signed-off-by: Manuel Bluhm --- nixos/modules/admin.nix | 131 ++++++++++++++-------------------------- nixos/tests/admin.nix | 46 +++++++------- nixos/tests/dbus.nix | 26 ++++---- nixos/tests/netvm.nix | 18 +++--- 4 files changed, 96 insertions(+), 125 deletions(-) diff --git a/nixos/modules/admin.nix b/nixos/modules/admin.nix index 1ec42bc..913cee6 100644 --- a/nixos/modules/admin.nix +++ b/nixos/modules/admin.nix @@ -16,9 +16,14 @@ let mkIf types trivial + strings concatStringsSep attrsets ; + inherit (import ./definitions.nix { inherit config lib; }) + transportSubmodule + tlsSubmodule + ; in { options.givc.admin = { @@ -31,22 +36,9 @@ in default = "localhost"; }; - addr = mkOption { - description = "IPv4 address."; - type = types.str; - default = "127.0.0.1"; - }; - - port = mkOption { - description = "Port of the admin service. Defaults to '9001'."; - type = types.str; - default = "9001"; - }; - - protocol = mkOption { - description = "Transport protocol, defaults to 'tcp'."; - type = types.str; - default = "tcp"; + addresses = mkOption { + description = "List of addresses for the admin service to listen on. Requires a list of 'transportSubmodule'. The host name should be inherited."; + type = types.listOf transportSubmodule; }; services = mkOption { @@ -64,38 +56,7 @@ in TLS options for gRPC connections. It is enabled by default to discourage unprotected connections, and requires paths to certificates and key being set. To disable it use 'tls.enable = false;'. ''; - type = - with types; - submodule { - options = { - enable = mkOption { - description = "Enable TLS. Defaults to 'true'."; - type = bool; - default = true; - }; - caCertPath = mkOption { - description = "Path to the CA certificate file."; - type = str; - default = ""; - }; - certPath = mkOption { - description = "Path to the service certificate file."; - type = str; - default = ""; - }; - keyPath = mkOption { - description = "Path to the service key file."; - type = str; - default = ""; - }; - }; - }; - default = { - enable = true; - caCertPath = ""; - certPath = ""; - keyPath = ""; - }; + type = tlsSubmodule; }; }; @@ -108,43 +69,45 @@ in } ]; - systemd.services.givc-admin = { - description = "GIVC admin module."; - enable = true; - after = [ "network-online.target" ]; - wants = [ "network-online.target" ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - Type = "exec"; - ExecStart = "${givc-admin}/bin/givc-admin"; - Restart = "always"; - RestartSec = 1; - }; - environment = - { - "NAME" = "${cfg.name}"; - "ADDR" = "${cfg.addr}"; - "PORT" = "${cfg.port}"; - "PROTO" = "${cfg.protocol}"; - "TYPE" = "4"; - "SUBTYPE" = "5"; - "TLS" = "${trivial.boolToString cfg.tls.enable}"; - "SERVICES" = "${concatStringsSep " " cfg.services}"; - } - // attrsets.optionalAttrs cfg.tls.enable { - "CA_CERT" = "${cfg.tls.caCertPath}"; - "HOST_CERT" = "${cfg.tls.certPath}"; - "HOST_KEY" = "${cfg.tls.keyPath}"; - } - // attrsets.optionalAttrs cfg.debug { - "RUST_BACKTRACE" = "1"; - "GIVC_LOG" = "debug"; - }; - }; - networking.firewall.allowedTCPPorts = + systemd.services.givc-admin = let - port = lib.strings.toInt cfg.port; + tcpAddresses = lib.filter (addr: addr.protocol == "tcp") cfg.addresses; + unixAddresses = lib.filter (addr: addr.protocol == "unix") cfg.addresses; + args = concatStringsSep " " ( + (map (addr: "--listen-tcp ${addr.addr}:${addr.port}") tcpAddresses) + ++ (map (addr: "--listen-unix ${addr.addr}") unixAddresses) + ); in - [ port ]; + { + description = "GIVC admin module."; + enable = true; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "exec"; + ExecStart = "${givc-admin}/bin/givc-admin ${args}"; + Restart = "always"; + RestartSec = 1; + }; + environment = + { + "NAME" = "${cfg.name}"; + "TYPE" = "4"; + "SUBTYPE" = "5"; + "TLS" = "${trivial.boolToString cfg.tls.enable}"; + "SERVICES" = "${concatStringsSep " " cfg.services}"; + } + // attrsets.optionalAttrs cfg.tls.enable { + "CA_CERT" = "${cfg.tls.caCertPath}"; + "HOST_CERT" = "${cfg.tls.certPath}"; + "HOST_KEY" = "${cfg.tls.keyPath}"; + } + // attrsets.optionalAttrs cfg.debug { + "RUST_BACKTRACE" = "1"; + "GIVC_LOG" = "debug"; + }; + }; + networking.firewall.allowedTCPPorts = map (addr: strings.toInt addr.port) cfg.addresses; }; } diff --git a/nixos/tests/admin.nix b/nixos/tests/admin.nix index 8780a9f..cde25e0 100644 --- a/nixos/tests/admin.nix +++ b/nixos/tests/admin.nix @@ -17,9 +17,14 @@ let }; admin = { name = "admin-vm"; - addr = addrs.adminvm; - port = "9001"; - protocol = "tcp"; # go version expect word "tcp" here, but it unused + addresses = [ + { + name = "admin-vm"; + addr = addrs.adminvm; + port = "9001"; + protocol = "tcp"; + } + ]; }; mkTls = name: { enable = tls; @@ -48,9 +53,7 @@ in enable = true; debug = true; inherit (admin) name; - inherit (admin) addr; - inherit (admin) port; - inherit (admin) protocol; + inherit (admin) addresses; tls = mkTls "admin-vm"; }; }; @@ -70,7 +73,7 @@ in port = "9000"; protocol = "tcp"; }; - inherit admin; + admin = lib.head admin.addresses; services = [ "microvm@admin-vm.service" "microvm@foot-vm.service" @@ -151,7 +154,7 @@ in ''; givc.sysvm = { enable = true; - inherit admin; + admin = lib.head admin.addresses; agent = { addr = addrs.guivm; name = "gui-vm"; @@ -208,7 +211,7 @@ in name = "chromium-vm"; addr = addrs.appvm; }; - inherit admin; + admin = lib.head admin.addresses; tls = mkTls "chromium-vm"; applications = [ { @@ -228,6 +231,7 @@ in let cli = "${self'.packages.givc-admin-rs.cli}/bin/givc-cli"; expected = "givc-ghaf-host.service"; # Name which we _expect_ to see registered in admin server's registry + adminAddr = lib.head admin.addresses; in # FIXME: why it so bizzare? (derived from name in cert) '' @@ -297,7 +301,7 @@ in time.sleep(1) # Ensure, that hostvm's agent registered in admin service. It take ~10 seconds to spin up and register itself - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} test ensure --retry 60 ${expected}")) + print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} test ensure --retry 60 ${expected}")) with subtest("setup gui vm"): # Ensure that sway in guiVM finished startup @@ -313,13 +317,13 @@ in time.sleep(5) # Give ssh some time to setup remote socket with subtest("set locale and timezone"): - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} set-locale en_US.UTF-8")) + print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} set-locale en_US.UTF-8")) adminvm.wait_for_file("/etc/locale-givc.conf") - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} set-timezone UTC")) + print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} set-timezone UTC")) adminvm.wait_for_file("/etc/timezone.conf") with subtest("Clean run"): - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm chromium-vm foot")) + print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} start --vm chromium-vm foot")) time.sleep(10) # Give few seconds to application to spin up wait_for_window("ghaf@appvm") @@ -328,34 +332,34 @@ in appvm.succeed("pkill foot") time.sleep(10) # .. then ask to restart - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm chromium-vm foot")) + print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} start --vm chromium-vm foot")) wait_for_window("ghaf@appvm") with subtest("pause/resume/stop application"): appvm.succeed("pgrep foot") - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} pause foot@1.service")) + print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} pause foot@1.service")) time.sleep(20) - js = hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} query-list --as-json 2>/dev/null") + js = hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} query-list --as-json 2>/dev/null") foot = by_name("foot@1.service", json.loads(js)) assert foot["status"] == "Paused" res = appvm.succeed("cat /sys/fs/cgroup/user.slice/user-1000.slice/user@1000.service/app.slice/app-foot.slice/foot@1.service/cgroup.events") assert "frozen 1" in res - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} resume foot@1.service")) + print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} resume foot@1.service")) time.sleep(20) res = appvm.succeed("cat /sys/fs/cgroup/user.slice/user-1000.slice/user@1000.service/app.slice/app-foot.slice/foot@1.service/cgroup.events") assert "frozen 0" in res - js = hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} query-list --as-json 2>/dev/null") + js = hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} query-list --as-json 2>/dev/null") foot = by_name("foot@1.service", json.loads(js)) assert foot["status"] == "Running" - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} stop foot@1.service")) + print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} stop foot@1.service")) appvm.fail("pgrep foot") with subtest("clear exit and restart"): - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm chromium-vm clearexit")) + print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} start --vm chromium-vm clearexit")) time.sleep(20) # Give few seconds to application to spin up, exit, then start it again - print(hostvm.succeed("${cli} --addr ${nodes.adminvm.config.givc.admin.addr} --port ${nodes.adminvm.config.givc.admin.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${nodes.adminvm.config.givc.admin.name} start --vm chromium-vm clearexit")) + print(hostvm.succeed("${cli} --addr ${adminAddr.addr} --port ${adminAddr.port} --cacert ${nodes.hostvm.givc.host.tls.caCertPath} --cert ${nodes.hostvm.givc.host.tls.certPath} --key ${nodes.hostvm.givc.host.tls.keyPath} ${if tls then "" else "--notls"} --name ${adminAddr.name} start --vm chromium-vm clearexit")) ''; }; }; diff --git a/nixos/tests/dbus.nix b/nixos/tests/dbus.nix index 444702e..358b3e7 100644 --- a/nixos/tests/dbus.nix +++ b/nixos/tests/dbus.nix @@ -16,11 +16,16 @@ let adminvm = "192.168.101.10"; appvm = "192.168.101.100"; }; - admin = { + adminConfig = { name = "admin-vm"; - addr = addrs.adminvm; - port = "9001"; - protocol = "tcp"; + addresses = [ + { + name = "admin-vm"; + addr = addrs.adminvm; + port = "9001"; + protocol = "tcp"; + } + ]; }; mkTls = name: { enable = tls; @@ -48,10 +53,7 @@ in environment.systemPackages = [ pkgs.grpcurl ]; givc.admin = { enable = true; - inherit (admin) name; - inherit (admin) addr; - inherit (admin) port; - inherit (admin) protocol; + inherit (adminConfig) addresses; tls = mkTls "admin-vm"; debug = false; }; @@ -125,7 +127,7 @@ in givc.sysvm = { enable = true; - inherit admin; + admin = lib.head adminConfig.addresses; agent = { addr = addrs.guivm; name = "gui-vm"; @@ -211,7 +213,7 @@ in givc.sysvm = { enable = true; - inherit admin; + admin = lib.head adminConfig.addresses; agent = { addr = addrs.netvm; name = "net-vm"; @@ -289,7 +291,7 @@ in givc.sysvm = { enable = true; - inherit admin; + admin = lib.head adminConfig.addresses; agent = { addr = addrs.audiovm; name = "audio-vm"; @@ -377,7 +379,7 @@ in givc.appvm = { enable = true; - inherit admin; + admin = lib.head adminConfig.addresses; agent = { addr = addrs.appvm; name = "chromium-vm"; diff --git a/nixos/tests/netvm.nix b/nixos/tests/netvm.nix index 3ac5862..d23d392 100644 --- a/nixos/tests/netvm.nix +++ b/nixos/tests/netvm.nix @@ -15,9 +15,14 @@ let }; admin = { name = "admin-vm"; - addr = addrs.adminvm; - port = "9001"; - protocol = "tcp"; # go version expect word "tcp" here, but it unused + addresses = [ + { + name = "admin-vm"; + addr = addrs.adminvm; + port = "9001"; + protocol = "tcp"; + } + ]; }; mkTls = name: { enable = tls; @@ -45,10 +50,7 @@ in environment.systemPackages = [ pkgs.grpcurl ]; givc.admin = { enable = true; - inherit (admin) name; - inherit (admin) addr; - inherit (admin) port; - inherit (admin) protocol; + inherit (admin) addresses; tls = mkTls "admin-vm"; }; }; @@ -171,7 +173,7 @@ in givc.sysvm = { enable = true; - inherit admin; + admin = lib.head admin.addresses; agent = { addr = addrs.netvm; name = "net-vm";