Migrate from guestfs to nbdfuse to save 400 MB

README.md

@@ -21,6 +21,7 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy
 
 - Linux machine with hardware virtualization support
 - kvm as hypervisor for the VMs (you can check through the `kvm-ok` command)
+- the fuse kernel module needs to be loaded (you can check through the `lsmod | grep fuse` command)
 - [docker](https://www.docker.com/) >= 24.x.y (for using kind and our deployment base image)
 - [kind](https://github.com/kubernetes-sigs/kind/releases) == v0.23.0 (for hosting the metal control plane)
 - [containerlab](https://containerlab.dev/install/) >= v0.56.0

images/sonic/Dockerfile

@@ -5,10 +5,13 @@ ENV LIBGUESTFS_BACKEND=direct
 RUN apt-get update && \
     apt-get --no-install-recommends install --yes \
         curl \
+        fuse \
+        fuse2fs \
+        libnbd-bin \
+        nbdkit \
         iproute2 \
-        linux-image-cloud-amd64 \
         python3 \
-        python3-guestfs \
+        qemu-utils \
         qemu-system-x86 \
         telnet
 

images/sonic/launch.py

@@ -1,19 +1,19 @@
 #!/usr/bin/python3
 import fcntl
+import glob
 import json
 import logging
 import os
+import shutil
 import signal
 import socket
 import struct
 import subprocess
 import sys
 import time
 
-import guestfs
-from guestfs import GuestFS
-
 BASE_IMG = '/sonic-vs.img'
+FUSE_PATH = '/mnt/sonic.img'
 
 
 class Qemu:
@@ -22,6 +22,8 @@ def __init__(self, name: str, smp: str, memory: str, interfaces: int):
         self._smp = smp
         self._memory = memory
         self._interfaces = interfaces
+        self._fuse = None
+        self._nbd = None
         self._p = None
         self._disk = '/overlay.img'
 
@@ -36,12 +38,33 @@ def prepare_overlay(self, base: str) -> None:
         ]
         subprocess.run(cmd, check=True)
 
-    def guestfs(self) -> GuestFS:
-        g = guestfs.GuestFS(python_return_dict=True)
-        g.add_drive_opts(filename=self._disk, format="qcow2", readonly=False)
-        g.launch()
-        g.mount('/dev/sda3', '/')
-        return g
+    def mount(self, path: str):
+        self._nbd = subprocess.Popen(['qemu-nbd', '--socket', '/nbd.sock', self._disk])
+
+        if self._nbd.poll() is not None:
+            print('qemu-nbd terminated with exit code ' + self._nbd.returncode)
+            exit(1)
+
+        nbdkit = ['nbdkit', '--single', 'nbd', 'socket=/nbd.sock', '--filter=partition', 'partition=3']
+        self._fuse = subprocess.Popen(['nbdfuse', FUSE_PATH, '--command'] + nbdkit)
+
+        while not os.path.exists(FUSE_PATH):
+            if self._fuse.poll() is not None:
+                print('nbdfuse terminated with exit code ' + self._fuse.returncode)
+                exit(1)
+            print("Waiting for file to exist...")
+            time.sleep(1)
+
+        subprocess.run(['fuse2fs', '-o', 'fakeroot', FUSE_PATH, path], check=True)
+
+    def umount(self, path: str):
+        subprocess.run(['fusermount', '-u', path], check=True)
+
+        self._fuse.terminate()
+        self._fuse.wait(timeout=10)
+
+        self._nbd.terminate()
+        self._nbd.wait(timeout=10)
 
     def start(self) -> None:
         cmd = [
@@ -72,41 +95,42 @@ def wait(self) -> None:
         self._p.wait()
 
 
-def initial_configuration(g: GuestFS) -> None:
-    image = g.glob_expand('/image-*')[0]
+def initial_configuration(path: str) -> None:
+    image = glob.glob(os.path.join(path, 'image-*'))[0]
 
-    g.rm(image + 'platform/firsttime')
+    os.remove(os.path.join(image, 'platform/firsttime'))
 
-    systemd_system = image + 'rw/etc/systemd/system/'
-    sonic_target_wants = systemd_system + 'sonic.target.wants/'
-    g.mkdir_p(sonic_target_wants)
+    systemd_system = os.path.join(image, 'rw/etc/systemd/system/')
+    sonic_target_wants = os.path.join(systemd_system, 'sonic.target.wants/')
+    os.makedirs(sonic_target_wants, exist_ok=True)
 
     # Copy frr-pythontools into the image
-    g.copy_in(localpath='/frr-pythontools.deb', remotedir=image + 'rw/')
+    shutil.copy('/frr-pythontools.deb', os.path.join(image, 'rw/'))
 
     # Workaround: Speed up lldp startup by remove hardcoded wait of 90 seconds
-    g.ln_s(linkname=systemd_system + 'aaastatsd.timer', target='/dev/null') # Radius
-    g.ln_s(linkname=systemd_system + 'featured.timer', target='/dev/null') # Feature handling not necessary
-    g.ln_s(linkname=systemd_system + 'hostcfgd.timer', target='/dev/null') # After boot Host configuration
-    g.ln_s(linkname=systemd_system + 'rasdaemon.timer', target='/dev/null') # After boot Host configuration
-    g.ln_s(linkname=systemd_system + 'tacacs-config.timer', target='/dev/null') # After boot Host configuration
+    os.symlink('/dev/null', os.path.join(systemd_system, 'aaastatsd.timer'))  # Radius
+    os.symlink('/dev/null', os.path.join(systemd_system, 'featured.timer'))  # Feature handling not necessary
+    os.symlink('/dev/null', os.path.join(systemd_system, 'hostcfgd.timer'))  # After boot Host configuration
+    os.symlink('/dev/null', os.path.join(systemd_system, 'rasdaemon.timer'))  # After boot Host configuration
+    os.symlink('/dev/null', os.path.join(systemd_system, 'tacacs-config.timer'))  # After boot Host configuration
     # Started by featured
-    g.ln_s(linkname=sonic_target_wants + 'lldp.service', target='/lib/systemd/system/lldp.service')
-    g.ln_s(linkname=systemd_system + 'pmon.service', target='/lib/systemd/system/pmon.service')
-    g.ln_s(linkname=sonic_target_wants + 'pmon.service', target='/lib/systemd/system/pmon.service')
+    os.symlink('/lib/systemd/system/lldp.service', os.path.join(sonic_target_wants, 'lldp.service'))
+    os.symlink('/lib/systemd/system/pmon.service', os.path.join(systemd_system, 'pmon.service'))
+    os.symlink('/lib/systemd/system/pmon.service', os.path.join(sonic_target_wants, 'pmon.service'))
 
     # Workaround: Only useful for BackEndToRRouter
-    g.ln_s(linkname=systemd_system + 'backend-acl.service', target='/dev/null')
+    os.symlink('/dev/null', os.path.join(systemd_system, 'backend-acl.service'))
 
     # Workaround: We don't need LACP
-    g.ln_s(linkname=systemd_system + 'teamd.service', target='/dev/null')
+    os.symlink('/dev/null', os.path.join(systemd_system, 'teamd.service'))
 
     # Workaround: Python module sonic_platform not present on vs images
-    g.ln_s(linkname=systemd_system + 'system-health.service', target='/dev/null')
-    g.ln_s(linkname=systemd_system + 'watchdog-control.service', target='/dev/null')
+    os.symlink('/dev/null', os.path.join(systemd_system, 'system-health.service'))
+    os.symlink('/dev/null', os.path.join(systemd_system, 'watchdog-control.service'))
+
+    etc_sonic = os.path.join(image, 'rw/etc/sonic/')
+    os.makedirs(etc_sonic, exist_ok=True)
 
-    etc_sonic = image + 'rw/etc/sonic/'
-    g.mkdir_p(etc_sonic)
     sonic_version = image.removeprefix('/image-').removesuffix('/')
     sonic_environment = f'''
         SONIC_VERSION=${sonic_version}
@@ -115,7 +139,8 @@ def initial_configuration(g: GuestFS) -> None:
         DEVICE_TYPE=LeafRouter
         ASIC_TYPE=vs
         '''.encode('utf-8')
-    g.write(path=etc_sonic + 'sonic-environment', content=sonic_environment)
+    with open(os.path.join(etc_sonic, 'sonic-environment'), mode='wb') as file:
+        file.write(sonic_environment)
 
     with open('/config_db.json') as f:
         config_db = json.load(f)
@@ -130,13 +155,14 @@ def initial_configuration(g: GuestFS) -> None:
     }
 
     config_db_json = json.dumps(config_db, indent=4, sort_keys=True)
-    g.write(path=etc_sonic + 'config_db.json', content=config_db_json.encode('utf-8'))
+    with open(os.path.join(etc_sonic, 'config_db.json'), mode='wb') as file:
+        file.write(config_db_json.encode('utf-8'))
 
     if os.path.exists('/authorized_keys'):
-        g.mkdir_p(image + 'rw/root/.ssh')
-        g.chmod(mode=0x0600, path=image + 'rw/root/.ssh')
-        g.copy_in(localpath='/authorized_keys', remotedir=image + 'rw/root/.ssh/')
-        g.chown(owner=0, group=0, path=image + 'rw/root/.ssh/authorized_keys')
+        os.makedirs(os.path.join(image, 'rw/root/.ssh'))
+        os.chmod(os.path.join(image, 'rw/root/.ssh'), 0o600)
+        shutil.copy('/authorized_keys', os.path.join(image, 'rw/root/.ssh/'))
+        os.chown(os.path.join(image, 'rw/root/.ssh/authorized_keys'), 0, 0)
 
 
 def main():
@@ -157,10 +183,10 @@ def main():
     vm.prepare_overlay(BASE_IMG)
 
     logger.info('Deploy initial config')
-    g = vm.guestfs()
-    initial_configuration(g)
-    g.shutdown()
-    g.close()
+    path = '/media'
+    vm.mount(path)
+    initial_configuration(path)
+    vm.umount(path)
 
     logger.info(f'Waiting for {interfaces} interfaces to be connected')
     wait_until_all_interfaces_are_connected(interfaces)