diff --git a/README.md b/README.md index 24e76bd..b0446c2 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy - Linux machine with hardware virtualization support - kvm as hypervisor for the VMs (you can check through the `kvm-ok` command) +- the fuse kernel module needs to be loaded (you can check through the `lsmod | grep fuse` command) - [docker](https://www.docker.com/) >= 24.x.y (for using kind and our deployment base image) - [kind](https://github.com/kubernetes-sigs/kind/releases) == v0.23.0 (for hosting the metal control plane) - [containerlab](https://containerlab.dev/install/) >= v0.56.0 diff --git a/images/sonic/Dockerfile b/images/sonic/Dockerfile index 324bdea..2fc870c 100644 --- a/images/sonic/Dockerfile +++ b/images/sonic/Dockerfile @@ -5,10 +5,13 @@ ENV LIBGUESTFS_BACKEND=direct RUN apt-get update && \ apt-get --no-install-recommends install --yes \ curl \ + fuse \ + fuse2fs \ + libnbd-bin \ + nbdkit \ iproute2 \ - linux-image-cloud-amd64 \ python3 \ - python3-guestfs \ + qemu-utils \ qemu-system-x86 \ telnet diff --git a/images/sonic/launch.py b/images/sonic/launch.py index f4921d4..e590b5f 100755 --- a/images/sonic/launch.py +++ b/images/sonic/launch.py @@ -1,8 +1,10 @@ #!/usr/bin/python3 import fcntl +import glob import json import logging import os +import shutil import signal import socket import struct @@ -10,10 +12,8 @@ import sys import time -import guestfs -from guestfs import GuestFS - BASE_IMG = '/sonic-vs.img' +FUSE_PATH = '/mnt/sonic.img' class Qemu: @@ -22,6 +22,8 @@ def __init__(self, name: str, smp: str, memory: str, interfaces: int): self._smp = smp self._memory = memory self._interfaces = interfaces + self._fuse = None + self._nbd = None self._p = None self._disk = '/overlay.img' @@ -36,12 +38,33 @@ def prepare_overlay(self, base: str) -> None: ] subprocess.run(cmd, check=True) - def guestfs(self) -> GuestFS: - g = guestfs.GuestFS(python_return_dict=True) - g.add_drive_opts(filename=self._disk, format="qcow2", readonly=False) - g.launch() - g.mount('/dev/sda3', '/') - return g + def mount(self, path: str): + self._nbd = subprocess.Popen(['qemu-nbd', '--socket', '/nbd.sock', self._disk]) + + if self._nbd.poll() is not None: + print('qemu-nbd terminated with exit code ' + self._nbd.returncode) + exit(1) + + nbdkit = ['nbdkit', '--single', 'nbd', 'socket=/nbd.sock', '--filter=partition', 'partition=3'] + self._fuse = subprocess.Popen(['nbdfuse', FUSE_PATH, '--command'] + nbdkit) + + while not os.path.exists(FUSE_PATH): + if self._fuse.poll() is not None: + print('nbdfuse terminated with exit code ' + self._fuse.returncode) + exit(1) + print("Waiting for file to exist...") + time.sleep(1) + + subprocess.run(['fuse2fs', '-o', 'fakeroot', FUSE_PATH, path], check=True) + + def umount(self, path: str): + subprocess.run(['fusermount', '-u', path], check=True) + + self._fuse.terminate() + self._fuse.wait(timeout=10) + + self._nbd.terminate() + self._nbd.wait(timeout=10) def start(self) -> None: cmd = [ @@ -72,41 +95,42 @@ def wait(self) -> None: self._p.wait() -def initial_configuration(g: GuestFS) -> None: - image = g.glob_expand('/image-*')[0] +def initial_configuration(path: str) -> None: + image = glob.glob(os.path.join(path, 'image-*'))[0] - g.rm(image + 'platform/firsttime') + os.remove(os.path.join(image, 'platform/firsttime')) - systemd_system = image + 'rw/etc/systemd/system/' - sonic_target_wants = systemd_system + 'sonic.target.wants/' - g.mkdir_p(sonic_target_wants) + systemd_system = os.path.join(image, 'rw/etc/systemd/system/') + sonic_target_wants = os.path.join(systemd_system, 'sonic.target.wants/') + os.makedirs(sonic_target_wants, exist_ok=True) # Copy frr-pythontools into the image - g.copy_in(localpath='/frr-pythontools.deb', remotedir=image + 'rw/') + shutil.copy('/frr-pythontools.deb', os.path.join(image, 'rw/')) # Workaround: Speed up lldp startup by remove hardcoded wait of 90 seconds - g.ln_s(linkname=systemd_system + 'aaastatsd.timer', target='/dev/null') # Radius - g.ln_s(linkname=systemd_system + 'featured.timer', target='/dev/null') # Feature handling not necessary - g.ln_s(linkname=systemd_system + 'hostcfgd.timer', target='/dev/null') # After boot Host configuration - g.ln_s(linkname=systemd_system + 'rasdaemon.timer', target='/dev/null') # After boot Host configuration - g.ln_s(linkname=systemd_system + 'tacacs-config.timer', target='/dev/null') # After boot Host configuration + os.symlink('/dev/null', os.path.join(systemd_system, 'aaastatsd.timer')) # Radius + os.symlink('/dev/null', os.path.join(systemd_system, 'featured.timer')) # Feature handling not necessary + os.symlink('/dev/null', os.path.join(systemd_system, 'hostcfgd.timer')) # After boot Host configuration + os.symlink('/dev/null', os.path.join(systemd_system, 'rasdaemon.timer')) # After boot Host configuration + os.symlink('/dev/null', os.path.join(systemd_system, 'tacacs-config.timer')) # After boot Host configuration # Started by featured - g.ln_s(linkname=sonic_target_wants + 'lldp.service', target='/lib/systemd/system/lldp.service') - g.ln_s(linkname=systemd_system + 'pmon.service', target='/lib/systemd/system/pmon.service') - g.ln_s(linkname=sonic_target_wants + 'pmon.service', target='/lib/systemd/system/pmon.service') + os.symlink('/lib/systemd/system/lldp.service', os.path.join(sonic_target_wants, 'lldp.service')) + os.symlink('/lib/systemd/system/pmon.service', os.path.join(systemd_system, 'pmon.service')) + os.symlink('/lib/systemd/system/pmon.service', os.path.join(sonic_target_wants, 'pmon.service')) # Workaround: Only useful for BackEndToRRouter - g.ln_s(linkname=systemd_system + 'backend-acl.service', target='/dev/null') + os.symlink('/dev/null', os.path.join(systemd_system, 'backend-acl.service')) # Workaround: We don't need LACP - g.ln_s(linkname=systemd_system + 'teamd.service', target='/dev/null') + os.symlink('/dev/null', os.path.join(systemd_system, 'teamd.service')) # Workaround: Python module sonic_platform not present on vs images - g.ln_s(linkname=systemd_system + 'system-health.service', target='/dev/null') - g.ln_s(linkname=systemd_system + 'watchdog-control.service', target='/dev/null') + os.symlink('/dev/null', os.path.join(systemd_system, 'system-health.service')) + os.symlink('/dev/null', os.path.join(systemd_system, 'watchdog-control.service')) + + etc_sonic = os.path.join(image, 'rw/etc/sonic/') + os.makedirs(etc_sonic, exist_ok=True) - etc_sonic = image + 'rw/etc/sonic/' - g.mkdir_p(etc_sonic) sonic_version = image.removeprefix('/image-').removesuffix('/') sonic_environment = f''' SONIC_VERSION=${sonic_version} @@ -115,7 +139,8 @@ def initial_configuration(g: GuestFS) -> None: DEVICE_TYPE=LeafRouter ASIC_TYPE=vs '''.encode('utf-8') - g.write(path=etc_sonic + 'sonic-environment', content=sonic_environment) + with open(os.path.join(etc_sonic, 'sonic-environment'), mode='wb') as file: + file.write(sonic_environment) with open('/config_db.json') as f: config_db = json.load(f) @@ -130,13 +155,14 @@ def initial_configuration(g: GuestFS) -> None: } config_db_json = json.dumps(config_db, indent=4, sort_keys=True) - g.write(path=etc_sonic + 'config_db.json', content=config_db_json.encode('utf-8')) + with open(os.path.join(etc_sonic, 'config_db.json'), mode='wb') as file: + file.write(config_db_json.encode('utf-8')) if os.path.exists('/authorized_keys'): - g.mkdir_p(image + 'rw/root/.ssh') - g.chmod(mode=0x0600, path=image + 'rw/root/.ssh') - g.copy_in(localpath='/authorized_keys', remotedir=image + 'rw/root/.ssh/') - g.chown(owner=0, group=0, path=image + 'rw/root/.ssh/authorized_keys') + os.makedirs(os.path.join(image, 'rw/root/.ssh')) + os.chmod(os.path.join(image, 'rw/root/.ssh'), 0o600) + shutil.copy('/authorized_keys', os.path.join(image, 'rw/root/.ssh/')) + os.chown(os.path.join(image, 'rw/root/.ssh/authorized_keys'), 0, 0) def main(): @@ -157,10 +183,10 @@ def main(): vm.prepare_overlay(BASE_IMG) logger.info('Deploy initial config') - g = vm.guestfs() - initial_configuration(g) - g.shutdown() - g.close() + path = '/media' + vm.mount(path) + initial_configuration(path) + vm.umount(path) logger.info(f'Waiting for {interfaces} interfaces to be connected') wait_until_all_interfaces_are_connected(interfaces)