From 19af1071522624a14ca64122f2cdbc6afd1decbd Mon Sep 17 00:00:00 2001 From: Yannick Heneault Date: Sat, 2 Apr 2022 21:30:03 -0400 Subject: [PATCH 1/3] added support for raspberry pi 4 --- share/product.mk | 72 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/share/product.mk b/share/product.mk index 221f073..bafaa58 100755 --- a/share/product.mk +++ b/share/product.mk @@ -25,6 +25,22 @@ DEBIAN = $(shell [ $(DISTRO) = 'debian' ] && echo 'y') FAB_ARCH = $(shell dpkg --print-architecture) I386 = $(shell [ $(FAB_ARCH) = 'i386' ] && echo 'y') AMD64 = $(shell [ $(FAB_ARCH) = 'amd64' ] && echo 'y') +ARM64 = $(shell [ $(FAB_ARCH) = 'arm64' ] && echo 'y') + +ifeq ($(I386),y) +ARCH_FAMILY=x86 +endif +ifeq ($(AMD64),y) +ARCH_FAMILY=x86 +endif +ifeq ($(ARM64),y) +ARCH_FAMILY=arm +# NONFREE is used to get raspi-firmware and firmware-brcm80211 +NONFREE=1 +endif +ifndef ARCH_FAMILY +$(error unsupported architecture) +endif ifdef FAB_POOL FAB_POOL_PATH=$(FAB_PATH)/pools/$(CODENAME) @@ -45,7 +61,7 @@ endif COMMON_PATCHES := turnkey.d $(COMMON_PATCHES) -CONF_VARS_BUILTIN ?= FAB_ARCH FAB_HTTP_PROXY I386 AMD64 RELEASE DISTRO CODENAME DEBIAN UBUNTU KERNEL DEBUG CHROOT_ONLY +CONF_VARS_BUILTIN ?= FAB_ARCH FAB_HTTP_PROXY I386 AMD64 ARM64 RELEASE DISTRO CODENAME DEBIAN UBUNTU KERNEL DEBUG CHROOT_ONLY define filter-undefined-vars $(foreach var,$1,$(if $($(var)), $(var))) @@ -136,8 +152,12 @@ endef ifdef CHROOT_ONLY all: root.sandbox else +ifeq ($(ARCH_FAMILY),arm) +all: $O/product.img +else all: $O/product.iso endif +endif define mount-deck @(deck $1 > /dev/null 2>&1) && echo deck $1 || true @@ -173,6 +193,7 @@ define help/body @echo ' CONF_VARS $(value CONF_VARS)' @echo @echo ' FAB_ARCH $(value FAB_ARCH)' + @echo ' ARCH_FAMILY $(value ARCH_FAMILY)' @echo ' FAB_POOL $(value FAB_POOL)' @echo ' FAB_POOL_PATH $(value FAB_POOL_PATH)' @echo ' FAB_PLAN_INCLUDE_PATH $(value FAB_PLAN_INCLUDE_PATH)/' @@ -222,7 +243,12 @@ define help/body @echo '# remake target and the targets that depend on it' @echo '$$ rm $(value STAMPS_DIR)/; make ' @echo - @echo '# build a target (default: product.iso)' + @if [ "$(ARCH_FAMILY)" = "arm" ]; \ + then \ + echo '# build a target (default: product.img)'; \ + else \ + echo '# build a target (default: product.iso)'; \ + fi @echo '$$ make [target] [O=path/to/build/dir]' @echo ' redeck # deck unmounted input/output decks (e.g., after reboot)' @echo @@ -231,19 +257,24 @@ define help/body @echo ' root.spec # the spec from which root.build is built (I.e., resolved plan)' @echo ' root.build # created by applying the root.spec to the bootstrap' @echo ' root.patched # deck root.build and apply the root overlay and removelist' - @echo + @echo @echo ' root.sandbox # changes (e.g., manual prototyping) inside the copy-on-write sandbox' @echo ' # saved as a separate, temporary cdroot squashfs overlay' @echo endef ifndef CHROOT_ONLY +ifeq ($(ARCH_FAMILY),arm) +help/body += ;\ + echo ' product.img \# product img for raspberry pi 4'; +else help/body += ;\ echo ' cdroot \# created by squashing root.patched into cdroot template + overlay'; \ echo ' product.iso \# product ISO created from the cdroot'; \ echo; \ echo ' updated-initramfs \# rebuild product with updated initramfs' endif +endif help: $(help/pre) @@ -255,7 +286,7 @@ define clean/body $(call remove-deck, $O/root.patched) $(call remove-deck, $O/root.build) $(call remove-deck, $O/bootstrap) - -rm -rf $O/root.spec $O/cdroot $O/product.iso $O/log $(STAMPS_DIR) + -rm -rf $O/root.spec $O/cdroot $O/product.iso $O/sdroot $O/product.img $O/product.img.xz $O/log $(STAMPS_DIR) endef clean: @@ -303,7 +334,7 @@ define run-conf-scripts if [ -n "$(wildcard $1/*)" ]; then \ echo "\$$(call $0,$1)"; \ fi; \ - for script in $1/*; do \ + for script in $1/* $1/$(ARCH_FAMILY).d/*; do \ [ -f "$$script" ] && [ -x "$$script" ] || continue; \ args_path=$(strip $1)/args/$$(echo $$(basename $$script) | sed 's/^[^a-zA-Z]*//'); \ args="$$([ -f $$args_path ] && (cat $$args_path | sed 's/#.*//'))"; \ @@ -337,7 +368,8 @@ define root.patched/body # apply the common overlays $(foreach overlay,$(_COMMON_OVERLAYS), @if echo $(overlay) | grep -q '\.d$$'; then \ - for d in $(overlay)/*; do \ + for d in $(overlay)/* $(overlay)/$(ARCH_FAMILY).d/*; do \ + if echo $$d | grep -q '\.d$$'; then continue; fi; \ echo fab-apply-overlay $$d $O/root.patched; \ fab-apply-overlay $$d $O/root.patched; \ done; \ @@ -494,6 +526,26 @@ define product.iso/body $(run-isohybrid) endef +# target: product.img +define product.img/body + qemu-img create -f raw $O/product.img 2G + parted -s $O/product.img mklabel msdos + parted -s $O/product.img -- mkpart primary fat32 4MiB 400MiB + parted -s $O/product.img -- mkpart primary ext2 400MiB 100% + kpartx -asv $O/product.img + mkfs -t vfat -n RASPIFIRM /dev/mapper/loop0p1 + mkfs -t ext4 -L RASPIROOT /dev/mapper/loop0p2 + mkdir -p $O/sdroot + mount /dev/mapper/loop0p2 $O/sdroot + mkdir -p $O/sdroot/boot/firmware + mount /dev/mapper/loop0p1 $O/sdroot/boot/firmware + cp -ax $O/root.sandbox/* $O/sdroot + umount $O/sdroot/boot/firmware + umount $O/sdroot + kpartx -dsv $O/product.img + xz -8 -f $O/product.img +endef + cdroot-dynamic: $(STAMPS_DIR)/root.sandbox $(cdroot-dynamic/pre) $(cdroot-dynamic/body) @@ -530,6 +582,14 @@ $O/product.iso: $(product.iso/deps) $(product.iso/deps/extra) product.iso: $O/product.iso +product.img/deps ?= $(STAMPS_DIR)/root.sandbox +$O/product.img: $(product.img/deps) $(product.img/deps/extra) + $(product.img/pre) + $(product.img/body) + $(product.img/post) + +product.img: $O/product.img + # target: updated-initramfs define updated-initramfs/body rm -rf $O/product.iso From 2fb251ef282ff238ce4aad09355af6aa31115cb4 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 9 May 2022 10:44:36 +0000 Subject: [PATCH 2/3] Update typing in fablib/installer.py. --- fablib/installer.py | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/fablib/installer.py b/fablib/installer.py index c86e01a..04093aa 100644 --- a/fablib/installer.py +++ b/fablib/installer.py @@ -11,12 +11,10 @@ import os from os.path import join, exists, basename import shutil -from typing import ( - Iterable, Optional, Dict, Tuple, List, TextIO, IO, AnyStr, cast -) +from typing import Iterable, Optional, TextIO, IO, AnyStr, cast import hashlib -import debian +from debian import debfile from chroot import Chroot from fablib import common @@ -108,7 +106,7 @@ def revert(self) -> None: class Installer: def __init__( self, chroot_path: str, - environ: Optional[Dict[str, str]]=None + environ: dict[str, str] = None ): if environ is None: environ = {} @@ -118,7 +116,7 @@ def __init__( self.chroot = Chroot(chroot_path, environ=env) @staticmethod - def _get_packages_priority(packages: List[str]) -> Tuple[List[str], List[str]]: + def _get_packages_priority(packages: list[str]) -> tuple[list[str], list[str]]: """high priority packages must be installed before regular packages APT should handle this, but in some circumstances it chokes... """ @@ -136,9 +134,9 @@ def _get_packages_priority(packages: List[str]) -> Tuple[List[str], List[str]]: return high, regular def _install( - self, packages: List[str], - ignore_errors: Optional[List[str]]=None, - extra_apt_args: Optional[List[str]]=None) -> None: + self, packages: list[str], + ignore_errors: list[str] = None, + extra_apt_args: list[str] = None) -> None: if ignore_errors is None: ignore_errors = [] @@ -177,7 +175,7 @@ def _install( f"apt-get {' '.join((args + packages))}") if apt_return_code != 0: - def get_last_log(path: str) -> List[str]: + def get_last_log(path: str) -> list[str]: log = [] with open(path) as fob: for line in fob: @@ -190,7 +188,7 @@ def get_last_log(path: str) -> List[str]: log.reverse() return log - def get_errors(log: List[str], error_str: str) -> List[str]: + def get_errors(log: list[str], error_str: str) -> list[str]: errors = [] for line in reversed(log): if line == error_str: @@ -228,7 +226,7 @@ def get_errors(log: List[str], error_str: str) -> List[str]: ) if errors: - for error in error: + for error in errors: common.error(error) raise Error('package installation errors') @@ -253,15 +251,15 @@ def get_errors(log: List[str], error_str: str) -> List[str]: os.remove(defer_log) def install( - self, packages: List[str], - ignore_errors: Optional[List[str]]=None) -> None: + self, packages: list[str], + ignore_errors: list[str] = None) -> None: raise NotImplementedError() class PoolInstaller(Installer): def __init__( self, chroot_path: str, pool_path: str, - arch: str, environ: Optional[Dict[str, str]]=None): + arch: str, environ: dict[str, str] = None): super(PoolInstaller, self).__init__(chroot_path, environ) from pool_lib import Pool @@ -270,7 +268,7 @@ def __init__( self.arch = arch @staticmethod - def _get_package_index(packagedir: str) -> List[str]: + def _get_package_index(packagedir: str) -> list[str]: def filesize(path: str) -> str: return str(os.stat(path).st_size) @@ -288,7 +286,7 @@ def sha256sum(path: str) -> str: # dl_path would best be calculated; but we don't have access to chroot_path here... dl_path = os.path.join("var/cache/apt/archives", package) if path.endswith(".deb"): - control = debian.debfile.DebFile(path).debcontrol() + control = debfile.DebFile(path).debcontrol() for field in list(control.keys()): index.append(field + ": " + control[field]) @@ -301,8 +299,8 @@ def sha256sum(path: str) -> str: return index def install( - self, packages: List[str], - ignore_errors: Optional[List[str]]=None + self, packages: list[str], + ignore_errors: list[str] = None ) -> None: """install packages into chroot via pool""" @@ -335,15 +333,15 @@ def install( class LiveInstaller(Installer): def __init__( self, chroot_path: str, - apt_proxy: Optional[str]=None, - environ: Optional[Dict[str, str]]=None): + apt_proxy: str = None, + environ: dict[str, str] = None): super(LiveInstaller, self).__init__(chroot_path, environ) self.apt_proxy = apt_proxy def install( - self, packages: List[str], - ignore_errors: Optional[List[str]]=None) -> None: + self, packages: list[str], + ignore_errors: list[str] = None) -> None: """install packages into chroot via live apt""" if ignore_errors is None: ignore_errors = [] From eb39b483b29eb687f08460589ed7463a5f9b34bc Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 9 May 2022 23:57:17 +0000 Subject: [PATCH 3/3] Use fstrings everywhere. --- fablib/installer.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fablib/installer.py b/fablib/installer.py index 04093aa..fbe7188 100644 --- a/fablib/installer.py +++ b/fablib/installer.py @@ -159,7 +159,7 @@ def _install( "#!/bin/sh", "echo", 'echo "Warning: Deferring update-initramfs $@"', - 'echo "update-initramfs $@" >> /%s' % defer_log, + f'echo "update-initramfs $@" >> /{defer_log}' ] fake_update_initramfs = RevertibleScript( join(self.chroot.path, "usr/sbin/update-initramfs"), lines @@ -220,10 +220,8 @@ def get_errors(log: list[str], error_str: str) -> list[str]: errors = set(errors) - set(ignore_errors) if ignored_errors: - print( - "Warning: ignoring package installation errors (%s)" - % " ".join(ignored_errors) - ) + print(f"Warning: ignoring package installation errors" + f" ({' '.join(ignored_errors)})") if errors: for error in errors: @@ -319,7 +317,7 @@ def install( print("deb file:/// local debs", file=cast(TextIO, sources_list)) sources_list.close() - index_file = "_dists_local_debs_binary-%s_Packages" % self.arch + index_file = f"_dists_local_debs_binary-{self.arch}_Packages" index_path = join(self.chroot.path, "var/lib/apt/lists", index_file) index = self._get_package_index(packagedir) with open(index_path, "w") as fob: