diff --git a/.gitignore b/.gitignore index a6883c2049..23bb4cf218 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ SKIP_IMAGES .pc *-pc apt-cacher-ng/ +cache/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2a5d8fe5f2..88f90c874d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,12 +5,9 @@ ENV DEBIAN_FRONTEND noninteractive RUN apt-get -y update && \ apt-get -y install --no-install-recommends \ - git vim parted \ + git vim parted apt-cacher-ng procps \ quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \ libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\ binfmt-support ca-certificates qemu-utils kpartx fdisk gpg pigz\ - && rm -rf /var/lib/apt/lists/* - -COPY . /pi-gen/ - -VOLUME [ "/pi-gen/work", "/pi-gen/deploy"] + && rm -rf /var/lib/apt/lists/* \ + && mkdir /pi-gen \ No newline at end of file diff --git a/README.md b/README.md index 2fdb65e252..9ad745b50d 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ To install the required dependencies for `pi-gen` you should run: ```bash apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \ dosfstools libarchive-tools libcap2-bin grep rsync xz-utils file git curl bc \ -qemu-utils kpartx gpg pigz +qemu-utils kpartx gpg pigz apt-cacher-ng ``` The file `depends` contains a list of tools needed. The format of this @@ -103,6 +103,20 @@ The following environment variables are supported: docker-compose up -d echo 'APT_PROXY=http://172.17.0.1:3142' >> config +* `ENABLE_CACHING` (Default: 0) + + Setting `ENABLE_CACHING` to `1` enables caching during the build process. + Cached data is stored in the `cache/` directory, allowing you to build the system with an internet connection and cache the required packages. + + To perform an offline build, set `USE_CACHED_DATA` to `1`. + This instructs `pi-gen` to use the cached packages stored in the `cache/` directory during the build, + eliminating the need for an internet connection. + + `USE_CACHED_DATA` (Default: 0) + + **CAUTION**: Please note that when `ENABLE_CACHING` is set to `1`, the `APT_PROXY` variable is + overridden and is not available for use during the build. + * `BASE_DIR` (Default: location of `build.sh`) **CAUTION**: Currently, changing this value will probably break build.sh diff --git a/build-docker.sh b/build-docker.sh index 83e76a587c..c83db0fc88 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -56,6 +56,8 @@ CONTAINER_NAME=${CONTAINER_NAME:-pigen_work} CONTINUE=${CONTINUE:-0} PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-0} PIGEN_DOCKER_OPTS=${PIGEN_DOCKER_OPTS:-""} +ENABLE_CACHING=${ENABLE_CACHING:-0} +USE_CACHED_DATA=${USE_CACHED_DATA:-0} if [ -z "${IMG_NAME}" ]; then echo "IMG_NAME not set in 'config'" 1>&2 @@ -91,7 +93,21 @@ case "$(uname -m)" in BASE_IMAGE=debian:bullseye ;; esac -${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" + +if [ "${USE_CACHED_DATA}" -eq 0 ]; then + ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}" +fi + +if [ "${USE_CACHED_DATA}" = "1" ]; then + image_found=$("${DOCKER}" images | grep -q 'pi-gen' && echo 1 || echo 0) + if [ "${image_found}" -eq 0 ]; then + echo "Local pi-gen image not found" + if [ ! -e cache/docker/pi-gen.tar ]; then + echo "No cached image found" + fi + "${DOCKER}" load -i cache/docker/pi-gen.tar + fi +fi if [ "${CONTAINER_EXISTS}" != "" ]; then DOCKER_CMDLINE_NAME="${CONTAINER_NAME}_cont" @@ -147,6 +163,7 @@ time ${DOCKER} run \ --cap-add=ALL \ -v /dev:/dev \ -v /lib/modules:/lib/modules \ + -v .:/pi-gen \ ${PIGEN_DOCKER_OPTS} \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ @@ -161,14 +178,15 @@ time ${DOCKER} run \ " & wait "$!" -# Ensure that deploy/ is always owned by calling user -echo "copying results from deploy/" -${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy - | tar -xf - +# echo "copying log from container ${CONTAINER_NAME} to deploy/" +# ${DOCKER} logs --timestamps "${CONTAINER_NAME}" & > deploy/build-docker.log -echo "copying log from container ${CONTAINER_NAME} to deploy/" -${DOCKER} logs --timestamps "${CONTAINER_NAME}" &>deploy/build-docker.log - -ls -lah deploy +# export image for offline usage +if [ "${ENABLE_CACHING}" = "1" ] && [ "$USE_CACHED_DATA" = "0" ]; then + echo "Exporting pi-gen image" + mkdir -p cache/docker + ${DOCKER} save -o cache/docker/pi-gen.tar pi-gen +fi # cleanup if [ "${PRESERVE_CONTAINER}" != "1" ]; then diff --git a/build.sh b/build.sh index 7d7bc40368..751435b33a 100755 --- a/build.sh +++ b/build.sh @@ -183,15 +183,6 @@ do esac done -term() { - if [ "${USE_QCOW2}" = "1" ]; then - log "Unloading image" - unload_qimage - fi -} - -trap term EXIT INT TERM - export PI_GEN=${PI_GEN:-pi-gen} export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen} export PI_GEN_RELEASE=${PI_GEN_RELEASE:-Raspberry Pi reference} @@ -201,6 +192,8 @@ if [ -z "${IMG_NAME}" ]; then exit 1 fi +export ENABLE_CACHING="${ENABLE_CACHING:-0}" +export USE_CACHED_DATA="${USE_CACHED_DATA:-0}" export USE_QEMU="${USE_QEMU:-0}" export IMG_DATE="${IMG_DATE:-"$(date +%Y-%m-%d)"}" export IMG_FILENAME="${IMG_FILENAME:-"${IMG_DATE}-${IMG_NAME}"}" @@ -270,6 +263,8 @@ source "${SCRIPT_DIR}/common" # shellcheck source=scripts/dependencies_check source "${SCRIPT_DIR}/dependencies_check" +source "${SCRIPT_DIR}/acng_handling" + export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" export USE_QCOW2="${USE_QCOW2:-0}" export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G} @@ -286,6 +281,22 @@ if [ "$SETFCAP" != "1" ]; then export CAPSH_ARG="--drop=cap_setfcap" fi +if [ "${ENABLE_CACHING}" = "1" ]; then + init_acng +fi + +term() { + if [ "${USE_QCOW2}" = "1" ]; then + log "Unloading image" + unload_qimage + fi + if [ "${ENABLE_CACHING}" = "1" ]; then + shutdown_acng + fi +} + +trap term EXIT INT TERM + dependencies_check "${BASE_DIR}/depends" #check username is valid diff --git a/scripts/acng_handling b/scripts/acng_handling new file mode 100644 index 0000000000..61b7af21ff --- /dev/null +++ b/scripts/acng_handling @@ -0,0 +1,45 @@ +#!/bin/bash + +source "${SCRIPT_DIR}/common" + +function write_config() { + echo \ + "CacheDir: $(pwd)/cache/apt-cacher-ng/data +LogDir: $(pwd)/cache/apt-cacher-ng/log +SupportDir: /usr/lib/apt-cacher-n +ReportPage: acng-report.html +Offlinemode: ${USE_CACHED_DATA} +Port:3143 +ExThreshold: 4 +LocalDirs: acng-doc /usr/share/doc/apt-cacher-ng" \ + > cache/apt-cacher-ng/conf/acng.conf +} + +function init_acng() { + mkdir -p cache/apt-cacher-ng/conf + mkdir -p cache/apt-cacher-ng/log + mkdir -p cache/apt-cacher-ng/data + write_config + apt-cacher-ng -c cache/apt-cacher-ng/conf & + sleep 1 + pid_acng=$(ps -A | grep 'apt-cacher-ng' | awk '{print $1}') + + if [ -z "${pid_acng}" ]; then + echo "apt-cacher-ng failed to start correctly" + exit 1 + fi + echo "Started apt-cacher-ng PID: ${pid_acng}" + + APT_PROXY="http://localhost:3143" +} + +export -f init_acng + +function shutdown_acng() { + if [ -n "${pid_acng}" ]; then + log "Stopping apt-cacher-ng" + kill ${pid_acng} + fi +} + +export -f shutdown_acng