refactor: small tidy up of kind deployer (#67)

This will be made similar across repos.
todo: put this in a common utils repo?

Signed-off-by: Tiago Castro <[email protected]>

scripts/k8s/deployer.sh

@@ -18,7 +18,9 @@ DOCKER="docker"
 HUGE_PAGES=1800
 SETUP_ZFS="false"
 SETUP_LVM="false"
-SETUP_MAYASTOR="false"
+SETUP_MAYASTOR="true"
+LABEL=
+SUDO=${SUDO:-"sudo"}
 
 help() {
   cat <<EOF
@@ -35,6 +37,7 @@ Options:
   --zfs                             Install ZFS utilities.
   --lvm                             Install LVM utilities and load required modules.
   --mayastor                        Setup pre-requisites, install and load required modules.
+  --label                           Label worker nodes with the io-engine selector.
 
 Command:
   start                             Start the k8s cluster.
@@ -64,10 +67,12 @@ while [ "$#" -gt 0 ]; do
       exit 0
       shift;;
     start)
+      [ -n "$COMMAND" ] && die "Command already specified"
       COMMAND="start"
       DO_ARGS="y"
       shift;;
     stop)
+      [ -n "$COMMAND" ] && die "Command already specified"
       COMMAND="stop"
       DO_ARGS="y"
       shift;;
@@ -97,33 +102,37 @@ while [ "$#" -gt 0 ]; do
       test $# -lt 1 && die "Missing hugepage number"
       HUGE_PAGES=$1
       shift;;
+    --label)
+      LABEL="true"
+      shift;;
+    --zfs)
+      SETUP_ZFS="true"
+      shift;;
+    --lvm)
+      SETUP_LVM="true"
+      shift;;
     --dry-run)
       if [ -z "$DRY_RUN" ]; then
         DRY_RUN="--dry-run"
         KIND="echo $KIND"
         FALLOCATE="echo $FALLOCATE"
         KUBECTL="echo $KUBECTL"
         DOCKER="echo $DOCKER"
+        SUDO="echo"
       fi
       shift;;
-    --zfs)
-      SETUP_ZFS="true"
-      shift;;
-    --lvm)
-      SETUP_LVM="true"
-      shift;;
     *)
-      die "Unknown argument $1!"
+      die "Unknown argument $1!\n$(help)"
       shift;;
   esac
 done
 
 if [ -z "$COMMAND" ]; then
-  die "No command specified!"
+  die "No command specified!\n$(help)"
 fi
 
 if [ "$COMMAND" = "stop" ]; then
-  sudo nvme disconnect-all
+  $SUDO nvme disconnect-all
   $KIND delete cluster
   exit 0
 fi
@@ -177,9 +186,6 @@ for node_index in $(seq 1 $WORKERS); do
     - hostPath: $TMP_KIND/$node
       containerPath: /var/local/mayastor
       propagation: HostToContainer
-    - hostPath: /
-      containerPath: /host
-      propagation: HostToContainer
 EOF
   if [ "$SETUP_ZFS" = "true" ]; then
     # Should already be installed by prereq script
@@ -190,17 +196,22 @@ EOF
 EOF
     chmod +x $TMP_KIND_ZFS
     cat <<EOF >> "$TMP_KIND_CONFIG"
+    - hostPath: /
+      containerPath: /host
+      propagation: HostToContainer
     - hostPath: $TMP_KIND_ZFS
       containerPath: /sbin/zfs
       propagation: HostToContainer
 EOF
   fi
 
   if [ "$SETUP_MAYASTOR" = "true" ]; then
-    cat <<EOF >> "$TMP_KIND_CONFIG"
+    if [ "$LABEL" = "true" ]; then
+      cat <<EOF >> "$TMP_KIND_CONFIG"
   labels:
     openebs.io/engine: mayastor
 EOF
+    fi
     mkdir -p $host_path/io-engine
     if [ -n "$POOL_SIZE" ]; then
       $FALLOCATE -l $POOL_SIZE $host_path/io-engine/disk.io

scripts/k8s/setup-io-prereq.sh

@@ -2,16 +2,20 @@
 
 set -e
 
-HUGE_PAGES=
+HUGE_PAGES=1800
 HUGE_PAGES_OVERRIDE=
-NVME_TCP=
 SETUP_ZFS=
 SETUP_LVM=
 SETUP_MAYASTOR=
 DRY_RUN=
-SYSCTL="sudo sysctl"
-MODPROBE="sudo modprobe"
+SUDO=${SUDO:-"sudo"}
+SYSCTL="sysctl"
+MODPROBE="modprobe"
+APTGET="apt-get"
+LSMOD="lsmod"
 UPDATED=0
+INSTALLED_KMODS=
+DISTRO=
 
 help() {
   cat <<EOF
@@ -40,56 +44,88 @@ die() {
 }
 
 setup_hugepages() {
-  $SYSCTL -w vm.nr_hugepages="$1"
-}
-
-modprobe_nvme_tcp() {
-  $MODPROBE nvme_tcp
+  $SUDO $SYSCTL -w vm.nr_hugepages="$1"
 }
 
 nvme_ana_check() {
   cat /sys/module/nvme_core/parameters/multipath
 }
 
 distro() {
-  cat /etc/os-release | awk -F= '/^NAME=/ {print $2}' | tr -d '"'
+  if [ -z "$DISTRO" ]; then
+    DISTRO=$(cat /etc/os-release | awk -F= '/^NAME=/ {print $2}' | tr -d '"')
+  fi
+  echo "$DISTRO"
 }
 
 install_kernel_modules_nsup() {
-  die "Installing kernel modules not supported for $1"
+  die "Installing extra kernel modules is not supported for $1"
 }
 
 update_apt() {
   if [ "$UPDATED" -eq 0 ]; then
-    sudo apt-get update
+    $SUDO $APTGET update
     UPDATED=1
   fi
 }
+apt_install() {
+  update_apt
+  if $SUDO $APTGET install -y $@; then
+    echo "Successfully installed $@"
+  else
+    die "Failed to install $@"
+  fi
+}
+
+kmod_loaded() {
+  local mod="${1//-/_}"
+  # either builtin or as a module
+  [ -d "/sys/module/$mod" ] || $LSMOD | grep -q "$mod"
+}
 
 install_kernel_modules() {
-  DISTRO="$(distro)"
-  case "$DISTRO" in
+  if [ "$INSTALLED_KMODS" = "yes" ]; then
+    return 0;
+  fi
+
+  case "$(distro)" in
     Ubuntu)
-      update_apt
-      sudo apt-get install -y linux-modules-extra-$(uname -r) xfsprogs quota
+      apt_install linux-modules-extra-$(uname -r)
+      INSTALLED_KMODS="yes"
       ;;
     NixOS | *)
-      install_kernel_modules_nsup "$DISTRO"
+      install_kernel_modules_nsup "$(distro)"
       ;;
   esac
 }
+modprobe_kmod() {
+  $SUDO $MODPROBE $@
+}
+
+load_kernel_module() {
+  if kmod_loaded $1; then
+    echo "$1 kernel module already loaded"
+    return 0;
+  fi
+
+  if ! modprobe_kmod $1 -q; then
+    # perhaps we're missing the modules?
+    install_kernel_modules
+    # now we can give it another go as our module may be available
+    if ! modprobe_kmod $1; then
+      die "Failed to load $1 kernel module!"
+    fi
+  fi
+
+  echo "$1 kernel module loaded"
+}
 
 install_zfs() {
-  if ! command -v zfs &> /dev/null; then
+  if ! command -v zfs &>/dev/null; then
     DISTRO="$(distro)"
     case "$DISTRO" in
       Ubuntu)
-        update_apt
-        if sudo apt-get install -y zfsutils-linux; then
-          echo "Successfully installed zfsutils-linux"
-        else
-          die "Failed to install zfsutils-linux"
-        fi
+        apt_install zfsutils-linux
         ;;
       NixOS | *)
         die "Installation of zfsutils-linux not supported for $DISTRO"
@@ -101,16 +137,11 @@ install_zfs() {
 }
 
 install_lvm() {
-  if ! command -v lvm &> /dev/null; then
+  if ! command -v lvm &>/dev/null; then
     DISTRO="$(distro)"
     case "$DISTRO" in
       Ubuntu)
-        update_apt
-        if sudo apt-get install -y lvm2; then
-          echo "Successfully installed lvm2"
-        else
-          die "Failed to install lvm2"
-        fi
+        apt_install lvm2
         ;;
       NixOS | *)
         die "Installation of lvm2 not supported for $DISTRO"
@@ -123,30 +154,32 @@ install_lvm() {
 
 load_lvm_modules() {
   # Load LVM snapshot and thin provisioning modules
-  sudo modprobe dm-snapshot
-  sudo modprobe dm-thin-pool
+  load_kernel_module dm-snapshot
+  load_kernel_module dm-thin-pool
 }
 
-mayastor() {
-if [ -n "$NVME_TCP" ]; then
-  if ! lsmod | grep "nvme_tcp" >/dev/null; then
-    if ! modprobe nvme_tcp >/dev/null; then
-      install_kernel_modules
-      if ! modprobe nvme_tcp; then
-        die "Failed to load nvme_tcp kernel module!"
-      fi
-    fi
-    echo "Installed nvme_tcp kernel module"
-  else
-    echo "nvme_tcp kernel module already installed"
-  fi
-
+setup_mayastor() {
+  load_kernel_module nvme-tcp
   if [ "$(nvme_ana_check)" != "Y" ]; then
     echo_stderr "NVMe multipath support is NOT enabled!"
   else
     echo "NVMe multipath support IS enabled"
   fi
-fi
+
+  if [ -n "$HUGE_PAGES" ]; then
+    pages=$($SYSCTL -b vm.nr_hugepages)
+
+    if [ "$HUGE_PAGES" -gt "$pages" ]; then
+      setup_hugepages "$HUGE_PAGES"
+    else
+      if [ "$HUGE_PAGES" -lt "$pages" ] && [ -n "$HUGE_PAGES_OVERRIDE" ]; then
+        echo "Overriding hugepages from $pages to $HUGE_PAGES, as requested"
+        setup_hugepages "$HUGE_PAGES"
+      else
+        echo "Current hugepages ($pages) are sufficient"
+      fi
+    fi
+  fi
 }
 
 while [ "$#" -gt 0 ]; do
@@ -171,7 +204,7 @@ while [ "$#" -gt 0 ]; do
       shift
       ;;
     --mayastor)
-      NVME_TCP="y"
+      SETUP_MAYASTOR="y"
       shift
       ;;
     --zfs)
@@ -185,8 +218,7 @@ while [ "$#" -gt 0 ]; do
     --dry-run)
       if [ -z "$DRY_RUN" ]; then
         DRY_RUN="--dry-run"
-        SYSCTL="echo $SYSCTL"
-        MODPROBE="echo $MODPROBE"
+        SUDO="echo $SUDO"
       fi
       shift
       ;;
@@ -196,23 +228,8 @@ while [ "$#" -gt 0 ]; do
   esac
 done
 
-if [ -n "$HUGE_PAGES" ]; then
-  pages=$(sysctl -b vm.nr_hugepages)
-
-  if [ "$HUGE_PAGES" -gt "$pages" ]; then
-    setup_hugepages "$HUGE_PAGES"
-  else
-    if [ "$HUGE_PAGES" -lt "$pages" ] && [ -n "$HUGE_PAGES_OVERRIDE" ]; then
-      echo "Overriding hugepages from $pages to $HUGE_PAGES, as requested"
-      setup_hugepages "$HUGE_PAGES"
-    else
-      echo "Current hugepages ($pages) are sufficient"
-    fi
-  fi
-fi
-
-if [ -n "$SETUP_MAYASTOR"]; then
-   mayastor
+if [ -n "$SETUP_MAYASTOR" ]; then
+  setup_mayastor
 fi
 
 if [ -n "$SETUP_ZFS" ]; then

shell.nix

@@ -44,7 +44,7 @@ mkShell {
   ;
 
   shellHook = ''
-    if [ -z "$CI" ]; then
+    if [ -z "$CI" ] && [ "$IN_NIX_SHELL" = "impure" ]; then
       pre-commit install
       pre-commit install --hook commit-msg
     fi