diff --git a/servers/cs306/user-vms/default.nix b/servers/cs306/user-vms/default.nix index 78c31cf..5a0df80 100644 --- a/servers/cs306/user-vms/default.nix +++ b/servers/cs306/user-vms/default.nix @@ -34,8 +34,11 @@ in acm.user-vms = { enable = true; users = builtins.fromJSON (builtins.readFile usersFile); - # Pin all CPU usages to the 4 host cores. - cpuPinning = [4 5 6 7]; + # Pin all CPU usages to the last 2 cores. + cpuPinning = [ + 2 3 # 3rd core + 6 7 # 4th core + ]; poolDirectory = "/var/lib/acm-vm"; }; diff --git a/user-machines/vm/default.nix b/user-machines/vm/default.nix index f78e06f..f8bce70 100644 --- a/user-machines/vm/default.nix +++ b/user-machines/vm/default.nix @@ -40,6 +40,12 @@ in default = "/var/lib/acm-vm"; description = "The directory to store VM disk images in."; }; + + volumeSize = mkOption { + type = types.str; + default = "4G"; + description = "The size of the VM disk images applied to new and existing images."; + }; virtConnection = mkOption { type = types.str; @@ -143,6 +149,61 @@ in sanitized_id = toLower (replaceStrings ["."] ["_"] user.id); }) self.users; + systemd.services.nixvirt-diskprep = { + serviceConfig.Type = "oneshot"; + description = "Prepare VM disk images for NixVirt"; + wantedBy = [ "multi-user.target" ]; + requires = [ "libvirtd.service" ]; + after = [ "libvirtd.service" ]; + path = with pkgs; [ + qemu + e2fsprogs + ]; + script = '' + set -euo pipefail + + export POOL_DIRECTORY=${self.poolDirectory} + export VOLUME_SIZE=${self.volumeSize} + images=( ${concatStringsSep "\n" (map (user: "${user.uuid}.img") self.users)} ) + + log() { echo "$@" >&2; } + + # Ensure the volume is created with the correct permissions. + umask 077 + + # Ensure the pool directory exists. + mkdir -p "$POOL_DIRECTORY" + + for image in ''${images[@]}; do + volumePath="$POOL_DIRECTORY/$image" + log "For volume $volumePath:" + + if [[ -f "$volumePath" ]]; then + log " volume already exists." + else + log " creating volume..." + # Clone the backing store image to a raw image then grow it. + qemu-img convert -O raw -S 0 ${ubuntu.image} "$volumePath" + # Disable copy-on-write for performance. + chattr +C "$volumePath" 2> /dev/null || { + log " couldn't disable copy-on-write, maybe the filesystem doesn't support it?" + } + fi + + log " resizing volume to $VOLUME_SIZE..." + qemu-img resize -f raw "$volumePath" "$VOLUME_SIZE" + + log " done!" + done + ''; + }; + + # Force NixVirt to run after the disk preparation service. + systemd.services.nixvirt.unitConfig = rec { + Requires = mkForce [ "libvirtd.service" "nixvirt-diskprep.service" ]; + After = mkForce Requires; + }; + virtualisation.libvirt.enable = true; virtualisation.libvirt.connections.${self.virtConnection} = { @@ -155,18 +216,6 @@ in type = "dir"; target.path = self.poolDirectory; }; - volumes = (map (user: { - present = !userIsDeleted user; - definition = virtlib.volume.writeXML { - name = "${user.uuid}.raw"; - capacity = { count = 4; unit = "GiB"; }; - target = { - format.type = "raw"; - permissions.mode = "0700"; - nocow = {}; - }; - }; - }) self.users); } ]; @@ -233,7 +282,7 @@ in # Allow 512BM total to the VM, but only allocate 128MB initially. # See https://pmhahn.github.io/virtio-balloon/. memory = { count = 512; unit = "MiB"; }; - currentMemory = { count = 150; unit = "MiB"; }; + currentMemory = { count = 256; unit = "MiB"; }; sysinfo = { type = "smbios"; @@ -268,22 +317,17 @@ in driver = { name = "qemu"; type = "raw"; - cache = "writethrough"; + cache = "writeback"; discard = "unmap"; }; source = { pool = "acm-vm-pool"; - volume = "${user.uuid}.raw"; + volume = "${user.uuid}.img"; }; target = { dev = "vda"; bus = "virtio"; }; - backingStore = { - type = "file"; - format.type = ubuntu.image.format; - source.file = "${ubuntu.image}"; - }; } { type = "file"; diff --git a/user-machines/vm/test-vm.nix b/user-machines/vm/test-vm.nix index ca417ae..d0ed732 100644 --- a/user-machines/vm/test-vm.nix +++ b/user-machines/vm/test-vm.nix @@ -9,7 +9,7 @@ in pkgs.testers.runNixOSTest { name = "user-machines-test"; - nodes.machine = { config, pkgs, ... }: { + nodes.machine = { config, pkgs, lib, ... }: { imports = [ ./. ]; @@ -42,6 +42,10 @@ pkgs.testers.runNixOSTest { # programs.virt-manager.enable = true; + # Accommodate the large VMs. + # Base 1GB + 4GB per user. + virtualisation.diskSize = 1024 + ((lib.length config.acm.user-vms.users) * 4 * 1024); + acm.user-vms = { enable = true; users = [