Skip to content

Commit

Permalink
armbian-next: artifacts: firmware/rootfs and actual usage of artifact…
Browse files Browse the repository at this point in the history
…s during image build

- artifacts: introduce `ARTIFACT_IGNORE_CACHE=yes`
- artifacts: introduce `DONT_BUILD_ARTIFACTS`, list of artifacts that if not found cached, fail the build
- kernel_package_source() is no more
- a long dissertation about kernels, families, and the universe
- artifacts: actually use rootfs artifact for image build
- artifacts: detangle via artifact_base_dir
- artifacts: rootfs: use folders in artifact_name; include cache_type
- artifacts: some cleanups / validations
- rootfs artifact; drop old rootfs cli
- artifacts: new CLI shortcuts; remove old firmware CLI
- artifacts: full firmware & usage
- use firmware artifacts in image build and install its debs
- kernel artifact sans legacy; use tmpdir for .deb target for all packages
- legacy artifact versions is no more; pack/unpack now in common obtain;
- artifacts: uboot: cleanup legacy renaming, use artifact version directly
- artifacts: add firmware (small) artifact
- deploy uboot to loop from artifact; allow tty to artifact; todos for cleaning
- fixes, kernel dtb/headers conditional; remove `.git` from Makefile url; use mapfile for finding files to hash
  - completely remove KERNEL_HAS_WORKING_HEADERS_FULL_SOURCE and `kernel_package_callback_linux_headers_full_source()`
  - don't use realpath for artifact_file_relative
  - curb some warnings
  - fix: only install headers & dtbs if such exist
- kernel .config hook modification hash versioning
- OCI_TARGET_BASE vs per-artifact defaults; only deploy to remote from CLI with OTB
- artifact revolver & installing into image
  - add artifact_map_packages and artifact_map_debs dicts
  - revolver accumulates all info
  - REPOSITORY_INSTALL is no more (for uboot/kernel, later others)
  - rename `VER` to `IMAGE_INSTALLED_KERNEL_VERSION`
  • Loading branch information
rpardini committed Feb 18, 2023
1 parent 8900eea commit 9bffa5e
Show file tree
Hide file tree
Showing 41 changed files with 1,258 additions and 1,070 deletions.
2 changes: 1 addition & 1 deletion extensions/flash-kernel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ function extension_prepare_config__prepare_flash_kernel() {
export BOOTSIZE=0 # No separate /boot, flash-kernel will "flash" the kernel+initrd to the firmware part.
export UEFI_MOUNT_POINT="/boot/firmware" # mount uefi partition at /boot/firmware
export CLOUD_INIT_CONFIG_LOCATION="/boot/firmware" # use /boot/firmware for cloud-init as well
export VER="${FK__PUBLISHED_KERNEL_VERSION}" # For the VERSION
export IMAGE_INSTALLED_KERNEL_VERSION="${FK__PUBLISHED_KERNEL_VERSION}" # For the VERSION
export EXTRA_BSP_NAME="${EXTRA_BSP_NAME}-fk${FK__PUBLISHED_KERNEL_VERSION}" # Unique bsp name.
}

Expand Down
10 changes: 5 additions & 5 deletions extensions/grub.sh
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ function extension_prepare_config__prepare_grub_standard() {
fi

if [[ "${DISTRO_GENERIC_KERNEL}" == "yes" ]]; then
export VER="${DISTRO_KERNEL_VER}"
export IMAGE_INSTALLED_KERNEL_VERSION="${DISTRO_KERNEL_VER}"
unset KERNELSOURCE # This should make Armbian skip most stuff. At least, I hacked it to.
export INSTALL_ARMBIAN_FIRMWARE=no # Should skip build and install of Armbian-firmware.
else
Expand Down Expand Up @@ -96,8 +96,8 @@ pre_umount_final_image__install_grub() {
if [[ "${UEFI_GRUB}" == "skip" ]]; then
display_alert "Skipping GRUB install" "due to UEFI_GRUB:${UEFI_GRUB}" "debug"
if [[ "${DISTRO_GENERIC_KERNEL}" == "yes" ]]; then
display_alert "Skipping GRUB install" "due to UEFI_GRUB:${UEFI_GRUB} - calling update_initramfs directly with VER=${DISTRO_KERNEL_VER}" "debug"
VER="${DISTRO_KERNEL_VER}" update_initramfs "${MOUNT}"
display_alert "Skipping GRUB install" "due to UEFI_GRUB:${UEFI_GRUB} - calling update_initramfs directly with IMAGE_INSTALLED_KERNEL_VERSION=${DISTRO_KERNEL_VER}" "debug"
IMAGE_INSTALLED_KERNEL_VERSION="${DISTRO_KERNEL_VER}" update_initramfs "${MOUNT}"
fi
return 0
fi
Expand All @@ -119,8 +119,8 @@ pre_umount_final_image__install_grub() {
cp "${SRC}"/packages/blobs/splash/grub.png "${MOUNT}"/usr/share/images/grub/wallpaper.png

if [[ "${DISTRO_GENERIC_KERNEL}" == "yes" ]]; then
display_alert "Using Distro Generic Kernel" "${EXTENSION}: update_initramfs with VER: ${DISTRO_KERNEL_VER}" "debug"
VER="${DISTRO_KERNEL_VER}" update_initramfs "${MOUNT}"
display_alert "Using Distro Generic Kernel" "${EXTENSION}: update_initramfs with IMAGE_INSTALLED_KERNEL_VERSION: ${DISTRO_KERNEL_VER}" "debug"
IMAGE_INSTALLED_KERNEL_VERSION="${DISTRO_KERNEL_VER}" update_initramfs "${MOUNT}"
fi

# Mount the chroot...
Expand Down
9 changes: 6 additions & 3 deletions extensions/lsmod.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,13 @@ function extension_prepare_config__prepare_localmodconfig() {
}

# This needs much more love than this. can be used to make "light" versions of kernels, that compile 3x-5x faster or more
function custom_kernel_config_post_defconfig__apply_localmodconfig() {
function custom_kernel_config__apply_localmodconfig() {
if [[ -f "${lsmod_file}" ]]; then
display_alert "${EXTENSION}: running localmodconfig on Kernel tree" "${LSMOD}" "warn"
run_kernel_make "LSMOD=${lsmod_file}" localmodconfig "> /dev/null" # quoted redirect to hide output even from logfile, it's way too long. stderr still shows
kernel_config_modifying_hashes+=("$(cat "${lsmod_file}")")
if [[ -f .config ]]; then
display_alert "${EXTENSION}: running localmodconfig on Kernel tree" "${LSMOD}" "warn"
run_kernel_make "LSMOD=${lsmod_file}" localmodconfig "> /dev/null" # quoted redirect to hide output even from logfile, it's way too long. stderr still shows
fi
else
display_alert "${EXTENSION}: lsmod file disappeared?" "${lsmod_file}" "err"
return 1 # exit with an error; this is not what the user expected
Expand Down
6 changes: 4 additions & 2 deletions extensions/nomod.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ function extension_prepare_config__prepare_localmodconfig() {
}

# This produces non-working kernels. It's meant for testing kernel image build and packaging.
function custom_kernel_config_post_defconfig__apply_mod2noconfig() {
run_kernel_make mod2noconfig
function custom_kernel_config__apply_mod2noconfig() {
kernel_config_modifying_hashes+=("mod2noconfig")
[[ -f .config ]] && run_kernel_make mod2noconfig
return 0 # short-circuit above
}
87 changes: 87 additions & 0 deletions lib/functions/artifacts/artifact-firmware.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
function artifact_firmware_prepare_version() {
artifact_version="undetermined" # outer scope
artifact_version_reason="undetermined" # outer scope

local ARMBIAN_FIRMWARE_SOURCE="${ARMBIAN_FIRMWARE_GIT_SOURCE:-"https://github.com/armbian/firmware"}"
local ARMBIAN_FIRMWARE_BRANCH="branch:${ARMBIAN_FIRMWARE_GIT_BRANCH:-"master"}"

debug_var ARMBIAN_FIRMWARE_SOURCE
debug_var ARMBIAN_FIRMWARE_BRANCH

declare short_hash_size=4

declare -A GIT_INFO=([GIT_SOURCE]="${ARMBIAN_FIRMWARE_SOURCE}" [GIT_REF]="${ARMBIAN_FIRMWARE_BRANCH}")
run_memoized GIT_INFO "git2info" memoized_git_ref_to_info
debug_dict GIT_INFO

declare fake_unchanging_base_version="1"

declare short_sha1="${GIT_INFO[SHA1]:0:${short_hash_size}}"

# get the hashes of the lib/ bash sources involved...
declare hash_files="undetermined"
calculate_hash_for_files "${SRC}"/lib/functions/compilation/packages/firmware-deb.sh "${SRC}"/lib/functions/artifacts/artifact-firmware.sh
declare bash_hash="${hash_files}"
declare bash_hash_short="${bash_hash:0:${short_hash_size}}"

# outer scope
artifact_version="${fake_unchanging_base_version}-SA${short_sha1}-B${bash_hash_short}"

declare -a reasons=(
"Armbian firmware git revision \"${GIT_INFO[SHA1]}\""
"framework bash hash \"${bash_hash}\""
)

artifact_version_reason="${reasons[*]}" # outer scope

artifact_map_packages=(
["armbian-firmware"]="armbian-firmware"
)

artifact_map_debs=(
["armbian-firmware"]="armbian-firmware_${artifact_version}_all.deb"
)

artifact_name="armbian-firmware"
artifact_type="deb"
artifact_base_dir="${DEB_STORAGE}"
artifact_final_file="${DEB_STORAGE}/armbian-firmware_${artifact_version}_all.deb"

return 0
}

function artifact_firmware_build_from_sources() {
FULL="" REPLACE="-full" LOG_SECTION="compile_firmware" do_with_logging compile_firmware
}

function artifact_firmware_cli_adapter_pre_run() {
declare -g ARMBIAN_COMMAND_REQUIRE_BASIC_DEPS="yes" # Require prepare_host_basic to run before the command.

# "gimme root on a Linux machine"
cli_standard_relaunch_docker_or_sudo
}

function artifact_firmware_cli_adapter_config_prep() {
declare KERNEL_ONLY="yes" # @TODO: this is a hack, for the board/family code's benefit...
use_board="no" prep_conf_main_minimal_ni < /dev/null # no stdin for this, so it bombs if tries to be interactive.
}

function artifact_firmware_get_default_oci_target() {
artifact_oci_target_base="ghcr.io/rpardini/armbian-release/"
}

function artifact_firmware_is_available_in_local_cache() {
is_artifact_available_in_local_cache
}

function artifact_firmware_is_available_in_remote_cache() {
is_artifact_available_in_remote_cache
}

function artifact_firmware_obtain_from_remote_cache() {
obtain_artifact_from_remote_cache
}

function artifact_firmware_deploy_to_remote_cache() {
upload_artifact_to_oci
}
94 changes: 94 additions & 0 deletions lib/functions/artifacts/artifact-full_firmware.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
function artifact_full_firmware_prepare_version() {
artifact_version="undetermined" # outer scope
artifact_version_reason="undetermined" # outer scope

local ARMBIAN_FIRMWARE_SOURCE="${ARMBIAN_FIRMWARE_GIT_SOURCE:-"https://github.com/armbian/firmware"}"
local ARMBIAN_FIRMWARE_BRANCH="branch:${ARMBIAN_FIRMWARE_GIT_BRANCH:-"master"}"

debug_var ARMBIAN_FIRMWARE_SOURCE
debug_var ARMBIAN_FIRMWARE_BRANCH
debug_var MAINLINE_FIRMWARE_SOURCE

declare short_hash_size=4

declare -A GIT_INFO=([GIT_SOURCE]="${ARMBIAN_FIRMWARE_SOURCE}" [GIT_REF]="${ARMBIAN_FIRMWARE_BRANCH}")
run_memoized GIT_INFO "git2info" memoized_git_ref_to_info
debug_dict GIT_INFO

declare -A GIT_INFO_MAINLINE=([GIT_SOURCE]="${MAINLINE_FIRMWARE_SOURCE}" [GIT_REF]="branch:main")
run_memoized GIT_INFO_MAINLINE "git2info" memoized_git_ref_to_info
debug_dict GIT_INFO_MAINLINE

declare fake_unchanging_base_version="1"

declare short_sha1="${GIT_INFO[SHA1]:0:${short_hash_size}}"
declare short_sha1_mainline="${GIT_INFO_MAINLINE[SHA1]:0:${short_hash_size}}"

# get the hashes of the lib/ bash sources involved...
declare hash_files="undetermined"
calculate_hash_for_files "${SRC}"/lib/functions/compilation/packages/firmware-deb.sh "${SRC}"/lib/functions/artifacts/artifact-firmware.sh
declare bash_hash="${hash_files}"
declare bash_hash_short="${bash_hash:0:${short_hash_size}}"

# outer scope
artifact_version="${fake_unchanging_base_version}-SA${short_sha1}-SM${short_sha1_mainline}-B${bash_hash_short}"

declare -a reasons=(
"Armbian firmware git revision \"${GIT_INFO[SHA1]}\""
"Mainline firmware git revision \"${GIT_INFO_MAINLINE[SHA1]}\""
"framework bash hash \"${bash_hash}\""
)

artifact_version_reason="${reasons[*]}" # outer scope

artifact_map_packages=(
["armbian-firmware-full"]="armbian-firmware-full"
)

artifact_map_debs=(
["armbian-firmware-full"]="armbian-firmware-full_${artifact_version}_all.deb"
)

artifact_name="armbian-firmware-full"
artifact_type="deb"
artifact_base_dir="${DEB_STORAGE}"
artifact_final_file="${DEB_STORAGE}/armbian-firmware-full_${artifact_version}_all.deb"

return 0
}

function artifact_full_firmware_build_from_sources() {
FULL="-full" REPLACE="" LOG_SECTION="compile_firmware_full" do_with_logging compile_firmware
}

function artifact_full_firmware_cli_adapter_pre_run() {
declare -g ARMBIAN_COMMAND_REQUIRE_BASIC_DEPS="yes" # Require prepare_host_basic to run before the command.

# "gimme root on a Linux machine"
cli_standard_relaunch_docker_or_sudo
}

function artifact_full_firmware_cli_adapter_config_prep() {
declare KERNEL_ONLY="yes" # @TODO: this is a hack, for the board/family code's benefit...
use_board="no" prep_conf_main_minimal_ni < /dev/null # no stdin for this, so it bombs if tries to be interactive.
}

function artifact_full_firmware_get_default_oci_target() {
artifact_oci_target_base="ghcr.io/rpardini/armbian-release/"
}

function artifact_full_firmware_is_available_in_local_cache() {
is_artifact_available_in_local_cache
}

function artifact_full_firmware_is_available_in_remote_cache() {
is_artifact_available_in_remote_cache
}

function artifact_full_firmware_obtain_from_remote_cache() {
obtain_artifact_from_remote_cache
}

function artifact_full_firmware_deploy_to_remote_cache() {
upload_artifact_to_oci
}
Loading

0 comments on commit 9bffa5e

Please sign in to comment.