LUKS FDE support

.gitignore

@@ -0,0 +1 @@
+bkp

bin/fai-make-nfsroot

@@ -95,9 +95,10 @@ generic=0
 force=0
 usexz=1
 full=1
+rebuildtar=0;
 
 # option e currently does nothing
-while getopts aghervC:B:fkKpPUzsN opt ; do
+while getopts aghervC:B:fkKpPRUzsN opt ; do
     case "$opt" in
         a) adjust=1 ;;
         g) generic=1 ;;
@@ -115,6 +116,7 @@ while getopts aghervC:B:fkKpPUzsN opt ; do
         e) expert=1 ;; # a dummy option, that only fai-setup uses
         p) sshpreserve=1 ;;
         P) sshpreserve=2 ;;
+        R) rebuildtar=1 ;;
         s) full=0 ;; # do only install a smaller list of packages
         ?) exit 5 ;; # error in option parsing
     esac
@@ -331,17 +333,32 @@ create_base() {
     call_debootstrap $FAI_DEBOOTSTRAP
     $ROOTCMD apt-get clean
     rm -f $NFSROOT/etc/resolv.conf $NFSROOT/etc/hostname $NFSROOT/etc/udev/rules.d/70-persistent-net.rules
-    if [ $usexz -eq 0 ]; then
-        echo "Creating base.tar.gz"
-    else
-        echo "Creating base.tar.xz"
-    fi
-    tar --xattrs --acls --one-file-system -C $NFSROOT -cf $NFSROOT/var/tmp/base.tar --exclude etc/machine-id --exclude var/tmp/base.tar --exclude 'var/lib/apt/lists/*_*' .
-    if [ $usexz -eq 1 ]; then
-        nice xz -q $NFSROOT/var/tmp/base.tar &
-    else
-        nice gzip $NFSROOT/var/tmp/base.tar &
-    fi
+
+    build_basetar;
+}
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+build_basetar() {
+  tar_base;
+  pack_base;
+  exit 0;
+}
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+tar_base() {
+  echo "Creating base.tar archive.";
+  rm -f "$NFSROOT/var/tmp/base.tar";
+  tar --xattrs --acls --one-file-system -C "$NFSROOT" -cf "$NFSROOT/var/tmp/base.tar" --exclude etc/machine-id --exclude var/tmp/base.tar --exclude 'var/lib/apt/lists/*_*' .;
+}
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+pack_base() {
+  if [ $usexz -eq 1 ]; then
+      echo "Creating compressed archive \"base.tar.xz\"."
+      rm -f "$NFSROOT/var/tmp/base.tar.xz";
+      nice xz -q "$NFSROOT/var/tmp/base.tar" &
+  else
+      echo "Creating compressed archive \"base.tar.gz\"."
+      rm -f "$NFSROOT/var/tmp/base.tar.gz";
+      nice gzip "$NFSROOT/var/tmp/base.tar" &
+  fi
 }
 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 extract_base() {
@@ -607,6 +624,8 @@ trap "bad_exit" ERR
 
 [ $adjust = 1 ] && adjust_nfsroot
 
+[ $rebuildtar = 1 ] && build_basetar;
+
 # remove all kernels from nfsroot
 [ -n "$kremove" ] && {
     echo "Removing all kernels from NFSROOT."

conf/NFSROOT

@@ -1,7 +1,7 @@
 # package list for creating the NFSROOT
 
 PACKAGES install-norec FULL
-nfs-common fai-nfsroot rpcbind
+#nfs-common fai-nfsroot rpcbind
 rsync lftp
 lshw procinfo
 dump reiserfsprogs xfsprogs xfsdump btrfs-progs  dosfstools
@@ -18,6 +18,7 @@ ssh
 netselect
 mdadm
 #git # git consumes a lot of disk space on the FAI CD
+cryptsetup cryptsetup-initramfs ### requirements for LUKS1 FDE
 
 PACKAGES install-norec
 # dracut replaces live-boot and initramfs-tools

conf/nfsroot.conf

@@ -1,13 +1,16 @@
 # For a detailed description see nfsroot.conf(5)
 
 # "<suite> <mirror>" for debootstrap
-FAI_DEBOOTSTRAP="buster http://deb.debian.org/debian"
-FAI_ROOTPW='$1$kBnWcO.E$djxB128U7dMkrltJHPf6d1'
+FAI_DEBOOTSTRAP="buster https://deb.debian.org/debian"
+### Generate random and unknown password:
+####  pwgen -sync1 26 1 | mkpasswd -m sha-512 -s
+FAI_ROOTPW='$1$kBnWcO.E$djxB128U7dMkrltJHPf6d1' ### fai
 
 NFSROOT=/srv/fai/nfsroot
 TFTPROOT=/srv/tftp/fai
 NFSROOT_HOOKS=/etc/fai/nfsroot-hooks/
-FAI_DEBOOTSTRAP_OPTS="--exclude=wget"
+### arch=... requires qemu-user-static installed on host.
+FAI_DEBOOTSTRAP_OPTS="--exclude=wget,nfs-common,fai-nfsroot,rpcbind --arch=amd64 --include=ca-certificates"
 
 # Configuration space
 FAI_CONFIGDIR=/srv/fai/config

conf/sources.list

@@ -1,8 +1,15 @@
 # These lines should work for many sites
 
-deb http://deb.debian.org/debian buster main contrib non-free
-deb http://deb.debian.org/debian-security buster/updates main contrib non-free
-#deb http://deb.debian.org/debian buster-backports main contrib non-free
+#deb http://deb.debian.org/debian buster main contrib non-free
+#deb http://deb.debian.org/debian-security buster/updates main contrib non-free
+##deb http://deb.debian.org/debian buster-backports main contrib non-free
+
+## repository that may contain newer fai packages for buster
+#deb http://fai-project.org/download buster koeln
+
+deb [arch=amd64] https://deb.debian.org/debian buster main contrib non-free
+deb [arch=amd64] https://deb.debian.org/debian-security buster/updates main contrib non-free
+#deb [arch=amd64] https://deb.debian.org/debian buster-backports main contrib non-free
 
 # repository that may contain newer fai packages for buster
-deb http://fai-project.org/download buster koeln
+deb [arch=amd64] https://fai-project.org/download buster koeln

doc/fai-guide.txt

@@ -802,16 +802,24 @@ _disk_config/_::
 Configuration files for disk partitioning, software RAID, LVM and file system creation.
 
 _basefiles/_::
-Normally the file 'base.tar.xz' (located inside the nfsroot) is extracted on the install
-client after the new file systems are created and before package are
+Normally the file 'base.tar.xz' (located inside the nfsroot, i.e.
+'/srv/fai/nfsroot/var/tmp/base.tar.xz') is extracted on the install
+client after the new file systems are created and before any packages are
 installed. This is a minimal base image, created right after calling
 debootstrap during the creation of the nfsroot on the install
 server. If you want to install another distribution than the nfsroot
 is, you can put a tar file into the subdirectory 'basefiles/' and name
 it after a class. Then the command `ftar(8)` is used to extract the
 tar file based on the classes defined. Thus the file has to be named 'CLASS.tar.xz' not 'CLASS.base.tar.xz'. This is done in task
 _extrbase_. Use this if you want to install another distribution or
-version than that running during the installation.
+version than that running during the installation. By specifying a different
+basefile 'basefiles/CLASS.tar.xz' only files contained within this basefile will be
+extracted. If you want to supply additional files, or replace existing files, use
+the 'files/' directory structure facility mentioned below. You can also modify
+an existing basefile to match your needs, instead. Another alternative would be
+to modify the tree below '/srv/fai/nfsroot/', note however that this tree is used to create
+the installation medium and also 'base.tar.xz'. You may find 'fai-make-nfsroot(8)
+-R' useful for repacking 'base.tar.xz'.
 +
 This basefile can also be received based on FAI classes via HTTP/HTTPS or FTP
 by defining the variable +FAI_BASEFILEURL+. FAI will download a file
@@ -1139,10 +1147,11 @@ linuxlogo-
 
 === [[cscripts]] Customization scripts
 
-The command `fai-do-scripts(1)` is called to execute all scripts in
-this directory. If a directory with a class name exists, all scripts
-matching '^[0-9][0-9]*' are executed in alphabetical order. So it's
-possible to use scripts of different languages (shell, cfengine,
+The command `fai-do-scripts(1) <dir>` is called to execute all scripts in
+this '<dir>'. If a directory with a class name
+exists ('<dir>/<CLASS>/^[0-9][0-9]*'), all scripts
+matching '^[0-9][0-9]*' are executed in alphabetical order.
+ So it's possible to use scripts of different languages (shell, cfengine,
 Perl, Python, Ruby, expect,..) for one class.
 
 Thoses scripts write their output to different log files, depending on
@@ -1409,7 +1418,8 @@ The format of the disk configuration is similar to a fstab file.
 
 The partitioning tool `setup-storage(8)` performs all commands
 necessary for creating the disk partition layout, software RAID, LVM
-and for creating the file systems. Read the manual page of
+and for creating the file systems. Disks and partitions may easily be 
+referenced by disk1.1, disk2.4 etc. Read the manual page of
 `setup-storage(8)` for a detailed description and some examples of the
 format.
 
@@ -1681,7 +1691,7 @@ To easily create the installation CD, you can use the following command
 (for more tuning details see `fai-cd(8)`):
 
 ----
- faiserver# fai-cd -m <partialMirrorDir> fai-cd.iso
+ faiserver# fai-cd -m <partialMirrorDir> faiserver-cd.iso
 ----
 
 This installation CD contains all data needed for the
@@ -1697,7 +1707,7 @@ stick by just writing the content of the ISO file to your USB stick
 (here the stick is _/dev/sdf_).
 
 ----
- faiserver# dd if=fai-cd.iso of=/dev/sdf bs=1M
+ faiserver# dd if=faiserver-cd.iso of=/dev/sdf bs=1M
 ----
 
 This is no live CD of the install server.
@@ -2356,6 +2366,17 @@ you need to create specific config directories in '/etc' for FAI, like
 faiserver#fai-make-nfsroot -C /etc/fai-jessie
 ----
 
+- How to provide DNS service for a local/testing/development FAI server or
+  deployments without installing an extra service or server?
++
+Install 'socat' and tell it to listen on the bridge interface while forwarding
+any DNS traffic to the locally listening resolver (i.e. systemd-resolved)
+masking it as if the traffic originated from localhost:
+----
+# apt install socat
+# socat UDP-LISTEN:53,fork,reuseaddr,bind=192.168.33.1 UDP:127.0.0.53:53
+----
+
 
 === flag_reboot (FAI_FLAGS)
 

lib/fetch-basefile

@@ -6,8 +6,13 @@
 #
 # Try to download a file CLASSNAME.tar.gz (or tgz, or tar.xz,...) from an URL
 # CLASSNAME must match a FAI class
-# The URL must provide a listing of all files in there
-# e.g.: FAI_BASEFILEURL=https://fai-project.org/download/basefiles
+# The URL must provide a (directory) listing of all files available for
+# download.
+# Example URL: FAI_BASEFILEURL=https://fai-project.org/download/basefiles
+# Example listing (excerpt) for the URL specified:
+# [...]
+# BUSTER_MIPS.tar.xz      BUSTER64.tar.xz      BUSTER_MIPS64EL.tar.xz
+# [...]
 #
 # variables needed: $classes, $FAI, $FAI_BASEFILEURL
 

man/fai-make-nfsroot.8

@@ -67,6 +67,10 @@ Remove all kernels from NFSROOT.
 Install additional packages defined in /etc/fai/NFSROOT into the
 nfsroot. Also copies all kernels from the nfsroot to the directory $TFTPROOT.
 You can use this option, to upgrade all packages inside the nfsroot.
+Remember that the compressed base.tar archive is
+.B *not*
+rebuilt/compressed again.
+Use -R for that in a second step.
 .TP
 .B \-N
 Additionally install non-free packages defined in the NONFREE section of /etc/fai/NFSROOT.
@@ -79,6 +83,9 @@ the ssh host keys from inside the nfsroot.
 Preserve ssh configs including the keys of the root account. This will
 not preserve the ssh host keys inside the nfsroot.
 .TP
+.B \-R
+Rebuild and compress base.tar archive after manual modification.
+.TP
 .B \-s
 Create a smaller nfsroot. Do not install packages defined in /etc/fai/NFSROOT in the FULL section.
 .TP

man/setup-storage.8

@@ -17,7 +17,8 @@ Using FAI disk_config files,
 \fBsetup-storage\fP
 computes effective partition and volume sizes and executes the necessary
 commands to configure storage devices. It manages disk drives as well as RAID
-and LVM volumes plus encryption and tmpfs. It handles following
+and LVM volumes plus encryption and tmpfs. Disks and partitions are designated 
+by \fBdisk1.2\fP, \fBdisk3.1\fP etc. and may thus be referenced as such. It handles following
 filesystems: \fBext2/3/4\fP, \fBvFAT(FAT32)\fP, \fBmsdos(FAT16)\fP, \fBreiserFS\fP, \fBXFS\fP and
 \fBBTRFS\fP but is flexible enough to be extended to further types as well.
 Once the storage devices are prepared, an appropriate
@@ -124,7 +125,7 @@ overrides this.
 .B flag_initial
 This variable determines if partitions should be preserved when they
 are tagged with preserve_reinstall (see below for details). Normally set by the
-list of FAI flags (FAI_FLAGS).
+list of FAI flags (FAI_FLAG).
 .TP
 .B SS_IGNORE_VG
 This variable may contain a list of volume groups, that are ignored by
@@ -365,6 +366,44 @@ luks    -        /dev/md1       -       -
 .fi
 .PP
 
+.TP
+Plain disk + LUKS + LVM2 aka Full Disk Encryption (FDE) example
+.sp
+.nf
+.ta 10n 20n 30n 40n 50n
+
+disk_config disk1 disklabel:msdos bootable:1 fstabkey:uuid align-at:1M
+primary - 100%  -       -
+
+disk_config cryptsetup
+### FDE:
+#luks         - disk1.1 -       - lukscreateopts="--type luks1"
+luks:'passwd' - disk1.1 -       - lukscreateopts="--type luks1"
+
+disk_config lvm fstabkey:uuid
+vg vg1  disk1.1
+vg1-root     /  30%-     ext4    defaults,errors=remount-ro,noatime,rw
+.sp
+.fi
+.PP
+.IP \(bu
+When deploying GRUB and LUKS with full disk encryption remember
+using 'disklabel:msdos' for now. EFI-compatible 'disklabel:gpt' makes cryptsetup use
+LUKS2 for which there is no support in GRUB yet. If there is a fixed setup
+with 'disklabel:gpt' and LUKS2 already, there is a way to downgrade LUKS2 to
+LUKS1, this is feasible without data loss (at your own risk). See the config/disk_config/FDE template
+for how to convert LUKS headers. Once GRUB2 supports LUKS2, LUKS FDE support in FAI will be updated.
+The LUKS version can be enforced with 'lukscreateopts="--type luks1"'.
+You may want to try 'disklabel:gpt' in conjunction with 'lukscreateopts="--type luks1"'.
+.IP \(bu
+The resulting crypto device is called 'crypt_dev_sda1', contains an LVM2 setup
+using volume group 'vg1' and a root partition 'vg1-root'. The luks password
+is 'passwd' and must be included literally, variable substitution is not happening
+here. If you are going to use the FDE template there's a better way: just set $LUKSPW in
+config/class/FDE.var . A password is mandatory when deploying Full Disk
+Encryption or you won't be able to unlock your system at boot time (before GRUB
+menu).
+
 .TP
 tmpfs example
 .sp