Added shared memory multi-server feature

Enabled memsocket to run on the host in addition to VMs Added support for multiple server instances, each managing a unique set of client IDs Enforced exclusivity of client IDs across all servers Renamed some Nix configuration options for clarity Swapped -c (server mode) and -s (client mode) options Added -l option in server mode to define the list of managed client IDs Signed-off-by: Jaroslaw Kurowski <[email protected]>
tiiuae · Nov 28, 2024 · 37d3bb5 · 37d3bb5
1 parent 01bb7e3
commit 37d3bb5
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 31 deletions.
diff --git a/modules/hardware/common/shared-mem.nix b/modules/hardware/common/shared-mem.nix
@@ -27,7 +27,7 @@ in
       type = types.bool;
       default = false;
       description = mdDoc ''
-        Enables shared memory communication between virtual machines (VMs)
+        Enables shared memory communication between virtual machines (VMs) and the host
       '';
     };
     memSize = mkOption {
@@ -83,26 +83,26 @@ in
         Enables the memsocket functionality on the host system
       '';
     };
-    instancesCount = mkOption {
+    shmSlots = mkOption {
       type = types.int;
       default =
         if cfg.enable_host then (builtins.length cfg.vms_enabled) + 1 else builtins.length cfg.vms_enabled;
       description = mdDoc ''
         Number of memory slots allocated in the shared memory region
       '';
     };
-    serverSocketPath = mkOption {
+    clientSocketPath = mkOption {
       type = types.path;
-      default = "/run/user/${builtins.toString config.ghaf.users.accounts.uid}/memsocket-server.sock";
+      default = "/run/user/${builtins.toString config.ghaf.users.accounts.uid}/memsocket-client.sock";
       description = mdDoc ''
         Specifies the path of the listening socket, which is used by Waypipe 
-        or other server applications as the output socket in server mode for 
+        or other server applications as the output socket in its server mode for 
         data transmission
       '';
     };
-    clientSocketPath = mkOption {
+    serverSocketPath = mkOption {
       type = types.path;
-      default = "/run/user/${builtins.toString config.ghaf.users.accounts.uid}/memsocket-client.sock";
+      default = "/run/user/${builtins.toString config.ghaf.users.accounts.uid}/memsocket-server.sock";
       description = mdDoc ''
         Specifies the location of the output socket, which will connected to 
         in order to receive data from AppVMs. This socket must be created by 
@@ -142,7 +142,7 @@ in
       }
       (mkIf cfg.enable_host {
         environment.systemPackages = [
-          (pkgs.callPackage ../../../packages/memsocket { vms = cfg.instancesCount; })
+          (pkgs.callPackage ../../../packages/memsocket { inherit (cfg) shmSlots; })
         ];
       })
       {
@@ -151,7 +151,7 @@ in
             pidFilePath = "/tmp/ivshmem-server.pid";
             ivShMemSrv =
               let
-                vectors = toString (2 * cfg.instancesCount);
+                vectors = toString (2 * cfg.shmSlots);
               in
               pkgs.writeShellScriptBin "ivshmemsrv" ''
                 if [ -S ${cfg.hostSocketPath} ]; then
@@ -180,8 +180,8 @@ in
       {
         microvm.vms =
           let
-            memsocket = pkgs.callPackage ../../../packages/memsocket { vms = cfg.instancesCount; };
-            vectors = toString (2 * cfg.instancesCount);
+            memsocket = pkgs.callPackage ../../../packages/memsocket { inherit (cfg) shmSlots; };
+            vectors = toString (2 * cfg.shmSlots);
             makeAssignment = vmName: {
               ${vmName} = {
                 config = {
@@ -200,7 +200,7 @@ in
                     boot.extraModulePackages = [
                       (pkgs.linuxPackages.callPackage ../../../packages/memsocket/module.nix {
                         inherit (config.microvm.vms.${vmName}.config.config.boot.kernelPackages) kernel;
-                        vmCount = cfg.instancesCount;
+                        inherit (cfg) shmSlots;
                       })
                     ];
                     services = {
@@ -221,7 +221,9 @@ in
                           after = [ "labwc.service" ];
                           serviceConfig = {
                             Type = "simple";
-                            ExecStart = "${memsocket}/bin/memsocket -c ${cfg.clientSocketPath}";
+                            # option '-l -1': listen to all slots. If you want to run other servers
+                            # for some slots, provide a list of handled slots, e.g.: '-l 1,3,5'
+                            ExecStart = "${memsocket}/bin/memsocket -s ${cfg.serverSocketPath} -l -1";
                             Restart = "always";
                             RestartSec = "1";
                           };
@@ -237,7 +239,7 @@ in
                           description = "memsocket";
                           serviceConfig = {
                             Type = "simple";
-                            ExecStart = "${memsocket}/bin/memsocket -s ${cfg.serverSocketPath} ${builtins.toString vmIndex}";
+                            ExecStart = "${memsocket}/bin/memsocket -c ${cfg.clientSocketPath} ${builtins.toString vmIndex}";
                             Restart = "always";
                             RestartSec = "1";
                           };

diff --git a/modules/microvm/virtualization/microvm/common/waypipe.nix b/modules/microvm/virtualization/microvm/common/waypipe.nix
@@ -25,7 +25,7 @@ let
         if configHost.ghaf.shm.display then
           ''
             #!${pkgs.runtimeShell} -e
-            ${pkgs.waypipe}/bin/waypipe -s ${configHost.ghaf.shm.serverSocketPath} server "$@"
+            ${pkgs.waypipe}/bin/waypipe -s ${configHost.ghaf.shm.clientSocketPath} server "$@"
           ''
         else
           ''
@@ -76,7 +76,7 @@ in
           RestartSec = "1";
           ExecStart =
             if configHost.ghaf.shm.display then
-              "${pkgs.waypipe}/bin/waypipe --secctx \"${vm.name}\" ${waypipeBorder} -s ${configHost.ghaf.shm.clientSocketPath} client"
+              "${pkgs.waypipe}/bin/waypipe --secctx \"${vm.name}\" ${waypipeBorder} -s ${configHost.ghaf.shm.serverSocketPath} client"
             else
               "${pkgs.waypipe}/bin/waypipe --vsock --secctx \"${vm.name}\" ${waypipeBorder} -s ${toString waypipePort} client";
         };

diff --git a/overlays/custom-packages/qemu/0001-ivshmem-flat-memory-support.patch b/overlays/custom-packages/qemu/0001-ivshmem-flat-memory-support.patch
@@ -1,17 +1,17 @@
-From 13229d9e11eaf15eb6679be036364b9d0256f1c1 Mon Sep 17 00:00:00 2001
+From e830291535405f73b9b8d32e97b273b806ed3097 Mon Sep 17 00:00:00 2001
 From: Jaroslaw Kurowski <[email protected]>
-Date: Wed, 10 Jul 2024 15:21:07 +0400
+Date: Thu, 21 Nov 2024 14:19:07 +0400
 Subject: ivshmem flat memory support
 
 ---
- contrib/ivshmem-server/ivshmem-server.c |  6 +-
+ contrib/ivshmem-server/ivshmem-server.c |  7 ++-
  hw/i386/pc_q35.c                        |  2 +
  hw/misc/ivshmem.c                       | 74 ++++++++++++++++++++++---
  include/hw/misc/ivshmem.h               |  1 +
- 4 files changed, 71 insertions(+), 12 deletions(-)
+ 4 files changed, 72 insertions(+), 12 deletions(-)
 
 diff --git a/contrib/ivshmem-server/ivshmem-server.c b/contrib/ivshmem-server/ivshmem-server.c
-index 2f3c732..906c5d0 100644
+index 2f3c732..e7c7774 100644
 --- a/contrib/ivshmem-server/ivshmem-server.c
 +++ b/contrib/ivshmem-server/ivshmem-server.c
 @@ -11,6 +11,7 @@
@@ -36,6 +36,14 @@ index 2f3c732..906c5d0 100644
          g_free(filename);
      }
 
+@@ -347,6 +347,7 @@ ivshmem_server_start(IvshmemServer *server)
+
+     server->sock_fd = sock_fd;
+     server->shm_fd = shm_fd;
++    server->cur_id = 1;
+
+     return 0;
+
 diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
 index c7bc8a2..d76939e 100644
 --- a/hw/i386/pc_q35.c
@@ -254,5 +262,5 @@ index 433ef53..43aeab7 100644
 
  #endif /* IVSHMEM_H */
 -- 
-2.45.2
+2.47.0
 
diff --git a/packages/memsocket/default.nix b/packages/memsocket/default.nix
@@ -4,7 +4,7 @@
   stdenv,
   lib,
   debug ? false,
-  vms,
+  shmSlots,
   fetchFromGitHub,
   ...
 }:
@@ -14,11 +14,11 @@ stdenv.mkDerivation {
   src = fetchFromGitHub {
     owner = "tiiuae";
     repo = "shmsockproxy";
-    rev = "2357926b94ed12c050fdbfbfc0f248393a4c9ea1";
-    sha256 = "sha256-9KlHuVbe5qvjRUXj7oyJ1X7CLvqj7/OoVGDWRqpIY2s=";
+    rev = "1e52f40df4b6a93cdbdf3ad43302cf1f7c313bfa";
+    sha256 = "sha256-nWPDWMryM5yONOwQWcBu4VlHJ5Y0slT6FtNKbajxrmY=";
   };
 
-  CFLAGS = "-O2 -DVM_COUNT=" + (toString vms) + (if debug then " -DDEBUG_ON" else "");
+  CFLAGS = "-O2 -DSHM_SLOTS=" + (toString shmSlots) + (if debug then " -DDEBUG_ON" else "");
   sourceRoot = "source/app";
 
   installPhase = ''

diff --git a/packages/memsocket/module.nix b/packages/memsocket/module.nix
@@ -5,18 +5,18 @@
   lib,
   kernel,
   fetchFromGitHub,
-  vmCount,
+  shmSlots,
   ...
 }:
 stdenv.mkDerivation {
-  inherit vmCount;
+  inherit shmSlots;
   name = "ivshmem-driver-${kernel.version}";
 
   src = fetchFromGitHub {
     owner = "tiiuae";
     repo = "shmsockproxy";
-    rev = "2357926b94ed12c050fdbfbfc0f248393a4c9ea1";
-    sha256 = "sha256-9KlHuVbe5qvjRUXj7oyJ1X7CLvqj7/OoVGDWRqpIY2s=";
+    rev = "1e52f40df4b6a93cdbdf3ad43302cf1f7c313bfa";
+    sha256 = "sha256-nWPDWMryM5yONOwQWcBu4VlHJ5Y0slT6FtNKbajxrmY=";
   };
 
   sourceRoot = "source/module";
@@ -29,7 +29,7 @@ stdenv.mkDerivation {
   makeFlags = [
     "KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build"
     "MODULEDIR=$(out)/lib/modules/${kernel.modDirVersion}/kernel/drivers/char"
-    "CFLAGS_kvm_ivshmem.o=\"-DCONFIG_KVM_IVSHMEM_VM_COUNT=${builtins.toString vmCount}\""
+    "CFLAGS_kvm_ivshmem.o=\"-DCONFIG_KVM_IVSHMEM_SHM_SLOTS=${builtins.toString shmSlots}\""
   ];
 
   meta = with lib; {