From 18c512f66e0b078470d85ee23d1cf50bea54513b Mon Sep 17 00:00:00 2001 From: AttilaGombosER Date: Sat, 15 Jun 2024 16:01:12 +0200 Subject: [PATCH] Add multi platform image build to release workflow (#2) --- .github/workflows/python-release.yml | 18 +++++++++ Dockerfile | 21 +++++------ Makefile | 17 ++++----- README.md | 55 ++++++++++++++-------------- bin/apt-server.py | 12 +++--- debian/control | 2 +- docker-compose.yaml | 6 +-- service/apt-server.service | 18 +++++++++ service/apt-server.service.template | 4 +- 9 files changed, 94 insertions(+), 59 deletions(-) create mode 100644 service/apt-server.service diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 85edfad..8d7e44a 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -23,5 +23,23 @@ jobs: with: debian-dist-type: 'application' debian-dist-command: 'make package' + - name: Set up QEMU for multi-architecture builds + uses: docker/setup-qemu-action@v3 + - name: Setup Docker buildx for multi-architecture builds + uses: docker/setup-buildx-action@v3 + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: effectiverange/apt-server:latest + build-args: | + PACKAGE_ARCHS=armhf,arm64,amd64 - name: Release uses: EffectiveRange/version-release-github-action@v1 diff --git a/Dockerfile b/Dockerfile index 772bfa9..095f67e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,4 @@ -ARG SERVER_ARCH=$SERVER_ARCH -ARG IMAGE_REPO=${SERVER_ARCH}/debian -ARG IMAGE_VER=bullseye-slim - -FROM ${IMAGE_REPO}:${IMAGE_VER} +FROM debian:bullseye-slim RUN apt update && apt upgrade -y @@ -10,12 +6,15 @@ RUN apt update && apt upgrade -y COPY dist/*.deb /etc/apt-server/ RUN apt install -y /etc/apt-server/*.deb -ARG CLIENT_ARCH=$CLIENT_ARCH -ENV ARCHITECTURES=${CLIENT_ARCH} +# Copy keys +COPY tests/keys/* /etc/apt-server/keys/ + +# Set package architectures +ARG PACKAGE_ARCHS=$PACKAGE_ARCHS +ENV ARCHITECTURES=${PACKAGE_ARCHS} # Start apt-server CMD /opt/venvs/apt-server/bin/python3 /opt/venvs/apt-server/bin/apt-server.py \ - --architectures ${ARCHITECTURES} \ - --release-template /opt/venvs/apt-server/templates/Release.template \ - --signing-key-path /opt/venvs/apt-server/.gnupg/private-key.asc \ - --public-key-path /opt/venvs/apt-server/.gnupg/public-key.asc +--port 80 --architectures ${ARCHITECTURES} \ +--private-key-path /etc/apt-server/keys/private-key.asc \ +--public-key-path /etc/apt-server/keys/public-key.asc diff --git a/Makefile b/Makefile index 10ce2d0..b31ca6c 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ colon := : $(colon) := : IMG_TAG=latest +PACKAGE_ARCHS=armhf,arm64,amd64 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) .PHONY: arm64 amd64 package build-image create-service apt-server-arm64-image apt-server-amd64-image apt-server-arm64-service apt-server-amd64-service @@ -13,20 +14,16 @@ amd64: $(eval ARCH=amd64) $(eval DOCKER_ARCH=$(ARCH)) +clean: + rm -rf build dist *.egg-info + package: apt-get update && apt-get install -y build-essential debhelper devscripts equivs dh-virtualenv python3-virtualenv dpkg-buildpackage -us -ui -uc --buildinfo-option=-udist --buildinfo-option=-Odist/apt-server.buildinfo --changes-option=-udist --changes-option=-Odist/apt-server.changes build-image: - docker build $(ROOT_DIR) --file Dockerfile --tag effectiverange/er-apt-server-$(ARCH)$(:)$(IMG_TAG) --build-arg SERVER_ARCH=$(DOCKER_ARCH) --build-arg CLIENT_ARCH=armhf - -create-service: - @cat $(ROOT_DIR)/service/apt-server.service.template | TAG=$(IMG_TAG) ARCH=$(ARCH) DOCKER_ARCH=$(DOCKER_ARCH) envsubst - -apt-server-arm64-image: arm64 package build-image - -apt-server-amd64-image: amd64 package build-image + docker build $(ROOT_DIR) --file Dockerfile --tag effectiverange/apt-server$(:)$(IMG_TAG) --build-arg --build-arg PACKAGE_ARCHS=$(PACKAGE_ARCHS) -apt-server-arm64-service: arm64 create-service +apt-server-arm64-image: arm64 build-image -apt-server-amd64-service: amd64 create-service +apt-server-amd64-image: amd64 build-image diff --git a/README.md b/README.md index c724331..a8c0789 100644 --- a/README.md +++ b/README.md @@ -54,27 +54,29 @@ apt install apt-server_1.0.0_all.deb ```commandline $ bin/apt-server.py --help -usage: apt-server.py [-h] [--log-file LOG_FILE] [--log-level LOG_LEVEL] [-a ARCHITECTURES [ARCHITECTURES ...]] [-r REPOSITORY_DIR] [-d DEB_PACKAGE_DIR] [-t RELEASE_TEMPLATE] [-i SIGNING_KEY_ID] [-k SIGNING_KEY_PATH] [-p SIGNING_KEY_PASS] [-P PUBLIC_KEY_PATH] [--port PORT] +usage: apt-server.py [-h] [-f LOG_FILE] [-l LOG_LEVEL] [-a ARCHITECTURES] [-r REPOSITORY_DIR] [-d DEB_PACKAGE_DIR] [-t RELEASE_TEMPLATE] [-i KEY_ID] [-k PRIVATE_KEY_PATH] [-p PRIVATE_KEY_PASS] + [-P PUBLIC_KEY_PATH] [--port PORT] optional arguments: -h, --help show this help message and exit - --log-file LOG_FILE log file path (default: /var/log/effective-range/apt-server/apt-server.log) - --log-level LOG_LEVEL - logging level (default: None) - -a ARCHITECTURES [ARCHITECTURES ...], --architectures ARCHITECTURES [ARCHITECTURES ...] - list of package architectures (default: ['amd64']) + -f LOG_FILE, --log-file LOG_FILE + log file path (default: /var/log/effective-range/apt-server/apt-server.log) + -l LOG_LEVEL, --log-level LOG_LEVEL + logging level (default: info) + -a ARCHITECTURES, --architectures ARCHITECTURES + served package architectures (comma separated) (default: amd64) -r REPOSITORY_DIR, --repository-dir REPOSITORY_DIR repository root directory (default: /etc/apt-repo) -d DEB_PACKAGE_DIR, --deb-package-dir DEB_PACKAGE_DIR directory containing the debian packages (default: /opt/debs) -t RELEASE_TEMPLATE, --release-template RELEASE_TEMPLATE release template file to use (default: templates/Release.template) - -i SIGNING_KEY_ID, --signing-key-id SIGNING_KEY_ID - ID of key used for signing (default: C1AEE2EDBAEC37595801DDFAE15BC62117A4E0F3) - -k SIGNING_KEY_PATH, --signing-key-path SIGNING_KEY_PATH + -i KEY_ID, --key-id KEY_ID + ID of key pair used for signing and verifying the signature (default: C1AEE2EDBAEC37595801DDFAE15BC62117A4E0F3) + -k PRIVATE_KEY_PATH, --private-key-path PRIVATE_KEY_PATH path of key used for signing (default: tests/keys/private-key.asc) - -p SIGNING_KEY_PASS, --signing-key-pass SIGNING_KEY_PASS - passphrase of key used for signing (default: admin123) + -p PRIVATE_KEY_PASS, --private-key-pass PRIVATE_KEY_PASS + passphrase of key used for signing (default: test1234) -P PUBLIC_KEY_PATH, --public-key-path PUBLIC_KEY_PATH path of key used for verification (default: tests/keys/public-key.asc) --port PORT repository server port to listen on (default: 9000) @@ -89,23 +91,22 @@ $ bin/apt-server.py -a armhf -t templates/Release.template -k tests/keys/private Output: ```commandline -2024-03-08T10:24:53.723322Z [info ] Started apt-server [AptServerMain] app_version=1.1.1 application=apt-server arguments={'architectures': ['armhf'], 'repository_dir': '/etc/apt-repo', 'deb_package_dir': '/opt/debs', 'release_template': '/opt/venvs/apt-server/templates/Release.template', 'signing_key_id': 'C1AEE2EDBAEC37595801DDFAE15BC62117A4E0F3', 'signing_key_path': '/opt/venvs/apt-server/.gnupg/private-key.asc', 'signing_key_pass': 'admin123', 'public_key_path': '/opt/venvs/apt-server/.gnupg/public-key.asc', 'port': 9000} hostname=er-debian -2024-03-08T10:24:53.725250Z [info ] Creating initial repository [AptServer] app_version=1.1.1 application=apt-server hostname=er-debian -2024-03-08T10:24:53.726090Z [info ] Creating pool directory [AptRepository] app_version=1.1.1 application=apt-server directory=/etc/apt-repo/pool hostname=er-debian -2024-03-08T10:24:53.726980Z [info ] Creating .deb package directory [AptRepository] app_version=1.1.1 application=apt-server directory=/opt/debs hostname=er-debian -2024-03-08T10:24:53.727728Z [info ] Linked .deb package directory [AptRepository] app_version=1.1.1 application=apt-server hostname=er-debian source=/opt/debs target=/etc/apt-repo/pool/main -2024-03-08T10:24:53.761824Z [info ] Generated Packages file [AptRepository] app_version=1.1.1 application=apt-server architecture=all file=/etc/apt-repo/dists/stable/main/binary-all/Packages hostname=er-debian -2024-03-08T10:24:53.796222Z [info ] Generated Packages file [AptRepository] app_version=1.1.1 application=apt-server architecture=armhf file=/etc/apt-repo/dists/stable/main/binary-armhf/Packages hostname=er-debian -2024-03-08T10:24:53.799556Z [info ] Generated Release file [AptRepository] app_version=1.1.1 application=apt-server file=/etc/apt-repo/dists/stable/Release hostname=er-debian -2024-03-08T10:24:53.800474Z [info ] Signing repository [AptRepository] app_version=1.1.1 application=apt-server hostname=er-debian -2024-03-08T10:24:55.088156Z [info ] Created signed Release file [AptSigner] app_version=1.1.1 application=apt-server file=/etc/apt-repo/dists/stable/InRelease hostname=er-debian -2024-03-08T10:24:55.448641Z [info ] Created signature file [AptSigner] app_version=1.1.1 application=apt-server file=/etc/apt-repo/dists/stable/Release.gpg hostname=er-debian -2024-03-08T10:24:55.449915Z [info ] Added public key file [AptSigner] app_version=1.1.1 application=apt-server file=/etc/apt-repo/dists/stable/public.key hostname=er-debian -2024-03-08T10:24:55.450723Z [info ] Watching directory for .deb file changes [AptServer] app_version=1.1.1 application=apt-server directory=/opt/debs hostname=er-debian -2024-03-08T10:24:55.451555Z [info ] Starting component [AptServer] app_version=1.1.1 application=apt-server component=file-observer hostname=er-debian -2024-03-08T10:24:55.452823Z [info ] Starting component [AptServer] app_version=1.1.1 application=apt-server component=web-server hostname=er-debian +2024-06-14T08:53:22.566892Z [info ] Started apt-server [AptServerApp] app_version=1.1.3 application=apt-server arguments={'log_file': '/var/log/effective-range/apt-server/apt-server.log', 'log_level': 'info', 'architecture': ['amd64'], 'repository_dir': '/etc/apt-repo', 'deb_package_dir': '/opt/debs', 'release_template': 'templates/Release.template', 'key_id': 'C1AEE2EDBAEC37595801DDFAE15BC62117A4E0F3', 'private_key_path': 'tests/keys/private-key.asc', 'private_key_pass': 'test1234', 'public_key_path': 'tests/keys/public-key.asc', 'port': 9000} hostname=Legion7iPro +2024-06-14T08:53:22.619221Z [info ] Creating initial repository [AptServer] app_version=1.1.3 application=apt-server hostname=Legion7iPro +2024-06-14T08:53:22.657566Z [info ] Removing existing link [AptRepository] app_version=1.1.3 application=apt-server hostname=Legion7iPro target=/etc/apt-repo/pool/main +2024-06-14T08:53:22.696269Z [info ] Linked .deb package directory [AptRepository] app_version=1.1.3 application=apt-server hostname=Legion7iPro source=/opt/debs target=/etc/apt-repo/pool/main +2024-06-14T08:53:22.791912Z [info ] Generated Packages file [AptRepository] app_version=1.1.3 application=apt-server architecture=all file=/etc/apt-repo/dists/stable/main/binary-all/Packages hostname=Legion7iPro +2024-06-14T08:53:22.885608Z [info ] Generated Packages file [AptRepository] app_version=1.1.3 application=apt-server architecture=amd64 file=/etc/apt-repo/dists/stable/main/binary-amd64/Packages hostname=Legion7iPro +2024-06-14T08:53:22.969732Z [info ] Generated Release file [AptRepository] app_version=1.1.3 application=apt-server file=/etc/apt-repo/dists/stable/Release hostname=Legion7iPro +2024-06-14T08:53:23.000795Z [info ] Signing initial repository [AptServer] app_version=1.1.3 application=apt-server hostname=Legion7iPro +2024-06-14T08:53:23.061975Z [info ] Added public key file [AptSigner] app_version=1.1.3 application=apt-server file=/etc/apt-repo/dists/stable/public.key hostname=Legion7iPro +2024-06-14T08:53:23.554509Z [info ] Created signed Release file [AptSigner] app_version=1.1.3 application=apt-server file=/etc/apt-repo/dists/stable/InRelease hostname=Legion7iPro +2024-06-14T08:53:24.038852Z [info ] Created signature file [AptSigner] app_version=1.1.3 application=apt-server file=/etc/apt-repo/dists/stable/Release.gpg hostname=Legion7iPro +2024-06-14T08:53:24.081987Z [info ] Watching directory for .deb file changes [AptServer] app_version=1.1.3 application=apt-server directory=/opt/debs hostname=Legion7iPro +2024-06-14T08:53:24.130686Z [info ] Starting component [AptServer] app_version=1.1.3 application=apt-server component=file-observer hostname=Legion7iPro +2024-06-14T08:53:24.173778Z [info ] Starting component [AptServer] app_version=1.1.3 application=apt-server component=web-server hostname=Legion7iPro ... -2024-03-08T12:01:56.750346Z [info ] Shutting down [AptServerMain] app_version=1.1.1 application=apt-server hostname=er-debian signum=2 +2024-06-14T08:54:10.480670Z [info ] Shutting down [AptServerApp] app_version=1.1.3 application=apt-server hostname=Legion7iPro signum=2 ``` ## Accessing the repository diff --git a/bin/apt-server.py b/bin/apt-server.py index 90bf034..f360e9f 100644 --- a/bin/apt-server.py +++ b/bin/apt-server.py @@ -39,11 +39,13 @@ def main() -> None: public_key_path = _get_absolute_path(resource_root, arguments.public_key_path) release_template_path = _get_absolute_path(resource_root, arguments.release_template) + architectures = arguments.architectures.split(',') + public_key = GpgKey(arguments.key_id, public_key_path) private_key = GpgKey(arguments.key_id, private_key_path, arguments.private_key_pass) apt_signer = ReleaseSigner(GPG(), public_key, private_key, repository_dir) - apt_repository = LinkedPoolAptRepository(APPLICATION_NAME, arguments.architectures, repository_dir, - deb_package_dir, release_template_path) + apt_repository = LinkedPoolAptRepository( + APPLICATION_NAME, architectures, repository_dir, deb_package_dir, release_template_path) handler_class = partial(SimpleHTTPRequestHandler, directory=repository_dir) web_server = HTTPServer(('', arguments.port), handler_class) @@ -62,10 +64,10 @@ def signal_handler(signum: int, frame: Any) -> None: def _get_argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--log-file', help='log file path', + parser.add_argument('-f', '--log-file', help='log file path', default='/var/log/effective-range/apt-server/apt-server.log') - parser.add_argument('--log-level', help='logging level', default='info') - parser.add_argument('-a', '--architectures', help='list of package architectures', nargs='+', default=['amd64']) + parser.add_argument('-l', '--log-level', help='logging level', default='info') + parser.add_argument('-a', '--architectures', help='served package architectures (comma separated)', default='amd64') parser.add_argument('-r', '--repository-dir', help='repository root directory', default='/etc/apt-repo') parser.add_argument('-d', '--deb-package-dir', help='directory containing the debian packages', default='/opt/debs') parser.add_argument('-t', '--release-template', help='release template file to use', diff --git a/debian/control b/debian/control index 02a395d..b0f1999 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Section: python Priority: extra Maintainer: Effective Range Build-Depends: debhelper (>= 9), python3, dh-virtualenv (>= 0.8) -Standards-Version: 3.9.5 +Standards-Version: 3.9.2 Package: apt-server Architecture: all diff --git a/docker-compose.yaml b/docker-compose.yaml index 6e6221d..7a0c78f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,8 +5,8 @@ services: build: dockerfile: Dockerfile args: - SERVER_ARCH: arm64v8 - CLIENT_ARCH: armhf + DOCKER_ARCH: arm64v8 + PACKAGE_ARCHS: armhf,arm64,amd64 stdin_open: true tty: true privileged: true @@ -19,4 +19,4 @@ services: build: dockerfile: Dockerfile args: - SERVER_ARCH: amd64 + DOCKER_ARCH: amd64 diff --git a/service/apt-server.service b/service/apt-server.service new file mode 100644 index 0000000..b9108ed --- /dev/null +++ b/service/apt-server.service @@ -0,0 +1,18 @@ +[Unit] +Description=APT Server Container +After=docker.service +Requires=docker.service +StartLimitIntervalSec=0 + +[Service] +Type=simple +Restart=always +RestartSec=10 +TimeoutStartSec=0 +ExecStartPre=-/usr/bin/docker pull effectiverange/apt-server:latest +ExecStart=/usr/bin/docker run --net=host --name=apt-server -v /opt/debs:/opt/debs -v /var/log/effective-range/apt-server:/var/log/effective-range/apt-server effectiverange/apt-server:latest +ExecStop=/usr/bin/docker kill apt-server +ExecStopPost=/usr/bin/docker rm apt-server + +[Install] +WantedBy=multi-user.target diff --git a/service/apt-server.service.template b/service/apt-server.service.template index ea754ea..9685572 100644 --- a/service/apt-server.service.template +++ b/service/apt-server.service.template @@ -9,8 +9,8 @@ Type=simple Restart=always RestartSec=10 TimeoutStartSec=0 -ExecStartPre=-/usr/bin/docker build /etc/apt-server --tag effectiverange/er-apt-server-${ARCH}:${TAG} --build-arg SERVER_ARCH=${DOCKER_ARCH} --build-arg CLIENT_ARCH=armhf -ExecStart=/usr/bin/docker run --net=host --name=apt-server -v /opt/debs:/opt/debs -v /var/log/effective-range/apt-server:/var/log/effective-range/apt-server ferenj/gss-apt-server-${ARCH}:${TAG} +ExecStartPre=-/usr/bin/docker pull effectiverange/apt-server:${TAG} +ExecStart=/usr/bin/docker run --net=host --name=apt-server -v /opt/debs:/opt/debs -v /var/log/effective-range/apt-server:/var/log/effective-range/apt-server effectiverange/apt-server:${TAG} ExecStop=/usr/bin/docker kill apt-server ExecStopPost=/usr/bin/docker rm apt-server