Add offline build functionality with apt-cacher-ng

.gitignore

@@ -7,3 +7,4 @@ SKIP_IMAGES
 .pc
 *-pc
 apt-cacher-ng/
+cache/

Dockerfile

@@ -5,12 +5,9 @@ ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get -y update && \
     apt-get -y install --no-install-recommends \
-        git vim parted \
+        git vim parted apt-cacher-ng procps \
         quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \
         libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\
         binfmt-support ca-certificates qemu-utils kpartx fdisk gpg pigz\
-    && rm -rf /var/lib/apt/lists/*
-
-COPY . /pi-gen/
-
-VOLUME [ "/pi-gen/work", "/pi-gen/deploy"]
+    && rm -rf /var/lib/apt/lists/* \
+    && mkdir /pi-gen

README.md

@@ -15,7 +15,7 @@ To install the required dependencies for `pi-gen` you should run:
 ```bash
 apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \
 dosfstools libarchive-tools libcap2-bin grep rsync xz-utils file git curl bc \
-qemu-utils kpartx gpg pigz
+qemu-utils kpartx gpg pigz apt-cacher-ng
 ```
 
 The file `depends` contains a list of tools needed.  The format of this
@@ -103,6 +103,20 @@ The following environment variables are supported:
        docker-compose up -d
        echo 'APT_PROXY=http://172.17.0.1:3142' >> config
 
+* `ENABLE_CACHING` (Default: 0)
+
+   Setting `ENABLE_CACHING` to `1` enables caching during the build process. 
+   Cached data is stored in the `cache/` directory, allowing you to build the system with an internet connection and cache the required packages.
+
+   To perform an offline build, set `USE_CACHED_DATA` to `1`. 
+   This instructs `pi-gen` to use the cached packages stored in the `cache/` directory during the build, 
+   eliminating the need for an internet connection.
+
+   `USE_CACHED_DATA` (Default: 0)
+
+   **CAUTION**: Please note that when `ENABLE_CACHING` is set to `1`, the `APT_PROXY` variable is 
+   overridden and is not available for use during the build.
+
  * `BASE_DIR`  (Default: location of `build.sh`)
 
    **CAUTION**: Currently, changing this value will probably break build.sh

build-docker.sh

@@ -56,6 +56,8 @@ CONTAINER_NAME=${CONTAINER_NAME:-pigen_work}
 CONTINUE=${CONTINUE:-0}
 PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-0}
 PIGEN_DOCKER_OPTS=${PIGEN_DOCKER_OPTS:-""}
+ENABLE_CACHING=${ENABLE_CACHING:-0}
+USE_CACHED_DATA=${USE_CACHED_DATA:-0}
 
 if [ -z "${IMG_NAME}" ]; then
 	echo "IMG_NAME not set in 'config'" 1>&2
@@ -91,7 +93,21 @@ case "$(uname -m)" in
     BASE_IMAGE=debian:bullseye
     ;;
 esac
-${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}"
+
+if [ "${USE_CACHED_DATA}" -eq 0 ]; then 
+  ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}"
+fi
+
+if [ "${USE_CACHED_DATA}" = "1" ]; then 
+  image_found=$("${DOCKER}" images | grep -q 'pi-gen' && echo 1 || echo 0)
+  if [ "${image_found}" -eq 0 ]; then 
+    echo "Local pi-gen image not found"
+    if [ ! -e cache/docker/pi-gen.tar ]; then
+      echo "No cached image found"
+    fi
+    "${DOCKER}" load -i cache/docker/pi-gen.tar
+  fi
+fi
 
 if [ "${CONTAINER_EXISTS}" != "" ]; then
   DOCKER_CMDLINE_NAME="${CONTAINER_NAME}_cont"
@@ -147,6 +163,7 @@ time ${DOCKER} run \
   --cap-add=ALL \
   -v /dev:/dev \
   -v /lib/modules:/lib/modules \
+  -v .:/pi-gen \
   ${PIGEN_DOCKER_OPTS} \
   --volume "${CONFIG_FILE}":/config:ro \
   -e "GIT_HASH=${GIT_HASH}" \
@@ -161,14 +178,15 @@ time ${DOCKER} run \
   " &
   wait "$!"
 
-# Ensure that deploy/ is always owned by calling user
-echo "copying results from deploy/"
-${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy - | tar -xf -
+# echo "copying log from container ${CONTAINER_NAME} to deploy/"
+# ${DOCKER} logs --timestamps "${CONTAINER_NAME}" & > deploy/build-docker.log
 
-echo "copying log from container ${CONTAINER_NAME} to deploy/"
-${DOCKER} logs --timestamps "${CONTAINER_NAME}" &>deploy/build-docker.log
-
-ls -lah deploy
+# export image for offline usage
+if [ "${ENABLE_CACHING}" = "1" ] && [ "$USE_CACHED_DATA" = "0" ]; then 
+  echo "Exporting pi-gen image"
+  mkdir -p cache/docker
+  ${DOCKER} save -o cache/docker/pi-gen.tar pi-gen
+fi
 
 # cleanup
 if [ "${PRESERVE_CONTAINER}" != "1" ]; then

build.sh

@@ -183,15 +183,6 @@ do
 	esac
 done
 
-term() {
-	if [ "${USE_QCOW2}" = "1" ]; then
-		log "Unloading image"
-		unload_qimage
-	fi
-}
-
-trap term EXIT INT TERM
-
 export PI_GEN=${PI_GEN:-pi-gen}
 export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen}
 export PI_GEN_RELEASE=${PI_GEN_RELEASE:-Raspberry Pi reference}
@@ -201,6 +192,8 @@ if [ -z "${IMG_NAME}" ]; then
 	exit 1
 fi
 
+export ENABLE_CACHING="${ENABLE_CACHING:-0}"
+export USE_CACHED_DATA="${USE_CACHED_DATA:-0}"
 export USE_QEMU="${USE_QEMU:-0}"
 export IMG_DATE="${IMG_DATE:-"$(date +%Y-%m-%d)"}"
 export IMG_FILENAME="${IMG_FILENAME:-"${IMG_DATE}-${IMG_NAME}"}"
@@ -270,6 +263,8 @@ source "${SCRIPT_DIR}/common"
 # shellcheck source=scripts/dependencies_check
 source "${SCRIPT_DIR}/dependencies_check"
 
+source "${SCRIPT_DIR}/acng_handling"
+
 export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"
 export USE_QCOW2="${USE_QCOW2:-0}"
 export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G}
@@ -286,6 +281,22 @@ if [ "$SETFCAP" != "1" ]; then
 	export CAPSH_ARG="--drop=cap_setfcap"
 fi
 
+if [ "${ENABLE_CACHING}" = "1" ]; then
+	init_acng
+fi
+
+term() { 
+	if [ "${USE_QCOW2}" = "1" ]; then
+		log "Unloading image"
+		unload_qimage
+	fi
+	if [ "${ENABLE_CACHING}" = "1" ]; then
+		shutdown_acng
+	fi
+}
+
+trap term EXIT INT TERM
+
 dependencies_check "${BASE_DIR}/depends"
 
 #check username is valid

scripts/acng_handling

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+source "${SCRIPT_DIR}/common"
+
+function write_config() {
+	echo \
+		"CacheDir: $(pwd)/cache/apt-cacher-ng/data 
+LogDir: $(pwd)/cache/apt-cacher-ng/log
+SupportDir: /usr/lib/apt-cacher-n
+ReportPage: acng-report.html
+Offlinemode: ${USE_CACHED_DATA}
+Port:3143
+ExThreshold: 4
+LocalDirs: acng-doc /usr/share/doc/apt-cacher-ng" \
+		> cache/apt-cacher-ng/conf/acng.conf
+}
+
+function init_acng() {
+	mkdir -p cache/apt-cacher-ng/conf 
+	mkdir -p cache/apt-cacher-ng/log
+	mkdir -p cache/apt-cacher-ng/data
+	write_config
+	apt-cacher-ng -c cache/apt-cacher-ng/conf  &
+	sleep 1
+	pid_acng=$(ps -A | grep 'apt-cacher-ng' | awk '{print $1}')
+
+	if [ -z "${pid_acng}" ]; then
+		echo "apt-cacher-ng failed to start correctly"
+		exit 1
+	fi
+	echo "Started apt-cacher-ng PID: ${pid_acng}"
+
+	APT_PROXY="http://localhost:3143"
+}
+
+export -f init_acng
+
+function shutdown_acng() {
+	if [ -n "${pid_acng}" ]; then
+		log "Stopping apt-cacher-ng"
+		kill ${pid_acng}
+	fi
+}
+
+export -f shutdown_acng