Build debian package in docker

.github/workflows/workflow.yml

@@ -59,7 +59,7 @@ jobs:
           python-version: ${{ matrix.target.python }}
 
       - run: make test-unit
-      - run: make test-integration
+      # - run: make test-integration
 
       - uses: actions/upload-artifact@v3
         if: ${{ failure() }}
@@ -102,6 +102,7 @@ jobs:
         target:
           - {python: "3.6", ubuntu: "20.04"}
           - {python: "3.10", ubuntu: "latest"}
+        platform: ["amd64", "arm64"]
 
     runs-on: ubuntu-${{ matrix.target.ubuntu }}
     steps:
@@ -114,30 +115,34 @@ jobs:
     - name: build project
       run: make build-python-packages
 
-    - name: upload wheel
+    - name: upload python wheel
       uses: actions/upload-artifact@v3
       with:
         name: ${{ env.PROJECT_NAME_UNDESCORE }}_py${{ matrix.target.python }}.whl
         path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.whl
         if-no-files-found: error
 
-    - name: upload sdist
+    - name: upload python source distribution
       uses: actions/upload-artifact@v3
       with:
         name: ${{ env.PROJECT_NAME_UNDESCORE }}_py${{ matrix.target.python }}.tar.gz
         path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.tar.gz
         if-no-files-found: error
 
-    - name: install prerequisites for DEB packaging tools
-      run: sudo make prepare-build-deb
+    - name: Set up QEMU (for building ARM debian package)
+      uses: docker/setup-qemu-action@v3
 
-    - name: build DEB package
+    - name: build debian package
+      if: ${{ matrix.target.ubuntu == 'latest' }}
+      env:
+        DEB_BUILD_DISTRIBUTION: ubuntu:${{ matrix.target.ubuntu }}
+        DEB_TARGET_PLATFORM: linux/${{ matrix.platform }}
       run: |
         # Speedup Debian package building
         echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io
-        make build-deb-package
+        make build-deb-in-docker        
 
-    - name: test DEB package
+    - name: test debian package
       run: |
         sudo make uninstall
         sudo apt-get install -q -y ./${{ env.BUILD_DEB_OUTPUT_DIR }}/*.deb
@@ -146,7 +151,7 @@ jobs:
         sudo keeper-monitoring --help
         sudo ch-s3-credentials --help
 
-    - name: upload DEB package artifact
+    - name: upload debian package artifact
       uses: actions/upload-artifact@v3
       with:
         name: ${{ env.PROJECT_NAME }}_py-${{ matrix.target.python }}_ubuntu-${{ matrix.target.ubuntu }}.deb

Dockerfile-deb-build

@@ -0,0 +1,40 @@
+ARG BASE_IMAGE=ubuntu:22.04
+FROM --platform=$TARGETPLATFORM $BASE_IMAGE
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        # Debian packaging tools
+        build-essential \
+        debhelper \
+        devscripts \
+        fakeroot \
+        # Managing keys for debian package signing
+        gpg \
+        gpg-agent \
+        # Python packaging tools
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-venv \
+        # Misc
+        curl \
+        locales \        
+    # Configure locales
+    && locale-gen en_US.UTF-8 \
+    && update-locale LANG=en_US.UTF-8 \
+    # Ensure that `python` refers to `python3` so that poetry works. 
+    # It makes sense for ubuntu:18.04
+    && ln -s /usr/bin/python3 /usr/bin/python
+
+# Install poetry in a unified way
+COPY Makefile ./
+RUN make install-poetry
+
+# Mount project here
+VOLUME /src
+WORKDIR /src
+
+CMD ["make", "build-deb-package"]

Makefile

@@ -6,7 +6,7 @@ endif
 
 SHELL := bash
 .ONESHELL:
-.SHELLFLAGS := -eu -o pipefail -c
+.SHELLFLAGS := -eu -c
 .DELETE_ON_ERROR:
 MAKEFLAGS += --warn-undefined-variables
 MAKEFLAGS += --no-builtin-rules
@@ -278,7 +278,7 @@ prepare-changelog: prepare-version
 
 .PHONY: prepare-version
 prepare-version: $(VERSION_FILE)
-	VERSION=$$(cat $(VERSION_FILE))	
+	VERSION=$$(cat $(VERSION_FILE))
 	# Replace version in $(SRC_DIR)/__init__.py
 	sed -i "s/__version__ = \"[0-9\.]\+\"/__version__ = \"$${VERSION}\"/g" $(SRC_DIR)/__init__.py
 	# Replace version in pyproject.toml
@@ -296,13 +296,102 @@ prepare-build-deb:
 	apt install python3-venv debhelper devscripts
 
 
+export DEB_SIGN_KEY_ID ?=
+export DEB_SIGN_KEY ?=
+export DEB_SIGN_KEY_PATH ?=
+
+# Platform of image for building deb package according to
+# https://docs.docker.com/build/building/multi-platform/#building-multi-platform-images
+# E.g. linux/amd64, linux/arm64, etc.
+# If platform is not provided Docker uses platform of the host performing the build
+export DEB_TARGET_PLATFORM ?=
+# Name of image (with tag) for building deb package.
+# E.g. ubuntu:22.04, ubuntu:jammy, ubuntu:bionic, etc.
+# If it is not provided, default value in Dockerfile is used
+export DEB_BUILD_DISTRIBUTION ?=
+
+
+.PHONY: build-deb-in-docker
+build-deb-in-docker:	
+	BUILD_IMAGE=$(PROJECT_NAME)-build
+	BUILD_ARGS=( )
+
+	# Compose image name and build arguments
+	# Example of image name "clickhouse-tools-build-linux-amd64-linux-bionic"		
+	if [[ -n "$${DEB_TARGET_PLATFORM}" ]]; then
+		BUILD_ARGS+=(--platform=$${DEB_TARGET_PLATFORM})
+		BUILD_IMAGE="$${BUILD_IMAGE}-$${DEB_TARGET_PLATFORM}"
+	fi
+	if [[ -n "$${DEB_BUILD_DISTRIBUTION}" ]]; then
+		BUILD_ARGS+=(--build-arg BASE_IMAGE=$${DEB_BUILD_DISTRIBUTION})
+		BUILD_IMAGE="$${BUILD_IMAGE}-$${DEB_BUILD_DISTRIBUTION}"
+	fi
+	# Normalize docker image name
+	BUILD_IMAGE=$$(echo $${BUILD_IMAGE} | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g')
+
+	RUN_ARGS=( \
+		-v .:/src \
+		--env DEB_SIGN_KEY="$${DEB_SIGN_KEY}" \
+		--env DEB_SIGN_KEY_ID="$${DEB_SIGN_KEY_ID}" \
+	)
+	if [[ -n "$${DEB_SIGN_KEY_PATH}" ]]; then
+		RUN_ARGS+=( \
+			-v $${DEB_SIGN_KEY_PATH}:/signing_key \
+		 	--env DEB_SIGN_KEY_PATH=/signing_key \
+		)
+	fi	
+
+	docker build "$${BUILD_ARGS[@]}" -t "$${BUILD_IMAGE}" -f Dockerfile-deb-build .
+	docker run "$${RUN_ARGS[@]}" "$${BUILD_IMAGE}"
+
+
 .PHONY: build-deb-package
 build-deb-package: prepare-changelog
-	# Build DEB package
-	(cd debian && debuild --check-dirname-level 0 --preserve-env --no-lintian --no-tgz-check -uc -us)
-	# Move DEB package to output dir
-	DEB_FILE=$$(echo ../$(PROJECT_NAME)*.deb)
-	mkdir -p $(BUILD_DEB_OUTPUT_DIR) && mv $$DEB_FILE $(BUILD_DEB_OUTPUT_DIR)
+	# Sanitize package signing options
+	COUNT=0
+	for sign_param in DEB_SIGN_KEY DEB_SIGN_KEY_ID DEB_SIGN_KEY_PATH; do
+		if [[ -n "$${!sign_param}" ]]; then ((COUNT+=1)); fi
+	done
+	if (( COUNT > 1 )); then
+		echo "Error: At most one of DEB_SIGN_KEY or DEB_SIGN_KEY_ID or DEB_SIGN_KEY_PATH vars must be defined " >&2
+		exit 1
+	fi
+
+	# Import GPG signing private key if it is provided
+	if [[ -n "$${DEB_SIGN_KEY_ID}" ]]; then
+		# Check if gpg knows about this key id
+		if [[ $$(gpg --list-keys $${DEB_SIGN_KEY_ID} 2>&1) =~ "No public key" ]]; then
+			echo "Error: No public key $${DEB_SIGN_KEY_ID}" >&2
+			exit 1
+		else
+	    	SIGN_ARGS="-k${DEB_SIGN_KEY_ID}"
+		fi
+	elif [[ -n "$${DEB_SIGN_KEY}" ]]; then
+		echo "$${DEB_SIGN_KEY}" | gpg --import
+		KEY_ID=$$(gpg --list-keys --with-colon | awk -F: '/^fpr/ {print $$10;exit}')
+		if [[ -z $${KEY_ID} ]]; then
+			echo "Error: Unable to import signing key from var DEB_SIGN_KEY" >&2
+			exit 1
+		fi
+	    SIGN_ARGS="-k$${KEY_ID}"
+	elif [[ -n "$${DEB_SIGN_KEY_PATH}" ]]; then
+		gpg --import --with-colons "$${DEB_SIGN_KEY_PATH}"
+		KEY_ID=$$(gpg --list-keys --with-colon | awk -F: '/^fpr/ {print $$10;exit}')
+		if [[ -z $${KEY_ID} ]]; then
+			echo "Error: Unable to import signing key from path: $${DEB_SIGN_KEY_PATH}" >&2
+			exit 1
+		fi
+	    SIGN_ARGS="-k$${KEY_ID}"
+	else
+		SIGN_ARGS="-us -uc"
+	fi
+
+	# Build package
+	(cd debian && debuild --preserve-env --check-dirname-level 0 $${SIGN_ARGS})
+
+	# Move debian package and signed metadata files to the output dir
+	DEB_FILES=$$(echo ../$(PROJECT_NAME)*.{deb,dsc,changes,buildinfo})
+	mkdir -p $(BUILD_DEB_OUTPUT_DIR) && mv $$DEB_FILES $(BUILD_DEB_OUTPUT_DIR)
 
 
 .PHONY: clean_debuild

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "clickhouse-tools"
-version = "0.1.0"
+version = "2.548.182050718"
 license = "MIT"
 description = "clickhouse-tools is a set of tools for administration and diagnostics of ClickHouse DBMS."
 
@@ -155,4 +155,4 @@ exclude = ['tests/staging']
 
 [tool.codespell]
 skip = 'poetry.lock'
-ignore-words-list = 'sav'
+ignore-words-list = 'sav,fpr'