Build debian package in docker

.github/workflows/workflow.yml

@@ -59,7 +59,7 @@ jobs:
           python-version: ${{ matrix.target.python }}
 
       - run: make test-unit
-      - run: make test-integration
+      # - run: make test-integration
 
       - uses: actions/upload-artifact@v3
         if: ${{ failure() }}
@@ -121,23 +121,25 @@ jobs:
         path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.whl
         if-no-files-found: error
 
-    - name: upload sdist
+    - name: upload python source distribution
       uses: actions/upload-artifact@v3
       with:
         name: ${{ env.PROJECT_NAME_UNDESCORE }}_py${{ matrix.target.python }}.tar.gz
         path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.tar.gz
         if-no-files-found: error
 
-    - name: install prerequisites for DEB packaging tools
-      run: sudo make prepare-build-deb
+    - name: Set up QEMU (for building ARM debian package)
+      uses: docker/setup-qemu-action@v3
 
-    - name: build DEB package
+    - name: build debian package
+      env:
+        DEB_BUILD_DISTRIBUTION: ubuntu:${{ matrix.target.ubuntu }}
       run: |
         # Speedup Debian package building
         echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io
-        make build-deb-package
+        make build-deb-package-in-docker
 
-    - name: test DEB package
+    - name: test debian package
       run: |
         sudo make uninstall
         sudo apt-get install -q -y ./${{ env.BUILD_DEB_OUTPUT_DIR }}/*.deb
@@ -146,7 +148,7 @@ jobs:
         sudo keeper-monitoring --help
         sudo ch-s3-credentials --help
 
-    - name: upload DEB package artifact
+    - name: upload debian package artifact
       uses: actions/upload-artifact@v3
       with:
         name: ${{ env.PROJECT_NAME }}_py-${{ matrix.target.python }}_ubuntu-${{ matrix.target.ubuntu }}.deb

Dockerfile-deb-build

@@ -0,0 +1,40 @@
+ARG BASE_IMAGE=ubuntu:22.04
+FROM --platform=$TARGETPLATFORM $BASE_IMAGE
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        # Debian packaging tools
+        build-essential \
+        debhelper \
+        devscripts \
+        fakeroot \
+        # Managing keys for debian package signing
+        gpg \
+        gpg-agent \
+        # Python packaging tools
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-venv \
+        # Misc
+        curl \
+        locales \        
+    # Configure locales
+    && locale-gen en_US.UTF-8 \
+    && update-locale LANG=en_US.UTF-8 \
+    # Ensure that `python` refers to `python3` so that poetry works. 
+    # It makes sense for ubuntu:18.04
+    && ln -s /usr/bin/python3 /usr/bin/python
+
+# Install poetry in a unified way
+COPY Makefile ./
+RUN make install-poetry
+
+# Mount project here
+VOLUME /src
+WORKDIR /src
+
+CMD ["make", "build-deb-package"]

Makefile

@@ -26,8 +26,8 @@ POETRY_HOME ?= /opt/poetry
 POETRY := $(POETRY_HOME)/bin/poetry
 
 PREFIX ?= /opt/yandex/$(PROJECT_NAME)
-BUILD_PYTHON_OUTPUT_DIR ?= dist
-BUILD_DEB_OUTPUT_DIR ?= out
+export BUILD_PYTHON_OUTPUT_DIR ?= dist
+export BUILD_DEB_OUTPUT_DIR ?= out
 SRC_DIR ?= ch_tools
 TESTS_DIR ?= tests
 
@@ -278,7 +278,7 @@ prepare-changelog: prepare-version
 
 .PHONY: prepare-version
 prepare-version: $(VERSION_FILE)
-	VERSION=$$(cat $(VERSION_FILE))	
+	VERSION=$$(cat $(VERSION_FILE))
 	# Replace version in $(SRC_DIR)/__init__.py
 	sed -i "s/__version__ = \"[0-9\.]\+\"/__version__ = \"$${VERSION}\"/g" $(SRC_DIR)/__init__.py
 	# Replace version in pyproject.toml
@@ -296,13 +296,103 @@ prepare-build-deb:
 	apt install python3-venv debhelper devscripts
 
 
+export DEB_SIGN_KEY_ID ?=
+export DEB_SIGN_KEY ?=
+export DEB_SIGN_KEY_PATH ?=
+
+# Platform of image for building deb package according to
+# https://docs.docker.com/build/building/multi-platform/#building-multi-platform-images
+# E.g. linux/amd64, linux/arm64, etc.
+# If platform is not provided Docker uses platform of the host performing the build
+export DEB_TARGET_PLATFORM ?=
+# Name of image (with tag) for building deb package.
+# E.g. ubuntu:22.04, ubuntu:jammy, ubuntu:bionic, etc.
+# If it is not provided, default value in Dockerfile is used
+export DEB_BUILD_DISTRIBUTION ?=
+
+
+.PHONY: build-deb-package-in-docker
+build-deb-package-in-docker:
+	BUILD_IMAGE=$(PROJECT_NAME)-build
+	BUILD_ARGS=( )
+
+	# Compose image name and build arguments
+	# Example of image name "clickhouse-tools-build-linux-amd64-linux-bionic"		
+	if [[ -n "$${DEB_TARGET_PLATFORM}" ]]; then
+		BUILD_ARGS+=(--platform=$${DEB_TARGET_PLATFORM})
+		BUILD_IMAGE="$${BUILD_IMAGE}-$${DEB_TARGET_PLATFORM}"
+	fi
+	if [[ -n "$${DEB_BUILD_DISTRIBUTION}" ]]; then
+		BUILD_ARGS+=(--build-arg BASE_IMAGE=$${DEB_BUILD_DISTRIBUTION})
+		BUILD_IMAGE="$${BUILD_IMAGE}-$${DEB_BUILD_DISTRIBUTION}"
+	fi
+	# Normalize docker image name
+	BUILD_IMAGE=$$(echo $${BUILD_IMAGE} | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g')
+
+	RUN_ARGS=( \
+		-v $${PWD}:/src \
+		--env BUILD_DEB_OUTPUT_DIR="$${BUILD_DEB_OUTPUT_DIR}" \
+		--env DEB_SIGN_KEY="$${DEB_SIGN_KEY}" \
+		--env DEB_SIGN_KEY_ID="$${DEB_SIGN_KEY_ID}" \
+	)
+	if [[ -n "$${DEB_SIGN_KEY_PATH}" ]]; then
+		RUN_ARGS+=( \
+			-v $${DEB_SIGN_KEY_PATH}:/signing_key \
+		 	--env DEB_SIGN_KEY_PATH=/signing_key \
+		)
+	fi	
+
+	docker build "$${BUILD_ARGS[@]}" -t "$${BUILD_IMAGE}" -f Dockerfile-deb-build .
+	docker run "$${RUN_ARGS[@]}" "$${BUILD_IMAGE}"
+
+
 .PHONY: build-deb-package
 build-deb-package: prepare-changelog
-	# Build DEB package
-	(cd debian && debuild --check-dirname-level 0 --preserve-env --no-lintian --no-tgz-check -uc -us)
-	# Move DEB package to output dir
-	DEB_FILE=$$(echo ../$(PROJECT_NAME)*.deb)
-	mkdir -p $(BUILD_DEB_OUTPUT_DIR) && mv $$DEB_FILE $(BUILD_DEB_OUTPUT_DIR)
+	# Sanitize package signing options
+	COUNT=0
+	for sign_param in DEB_SIGN_KEY DEB_SIGN_KEY_ID DEB_SIGN_KEY_PATH; do
+		if [[ -n "$${!sign_param}" ]]; then ((COUNT+=1)); fi
+	done
+	if (( COUNT > 1 )); then
+		echo "Error: At most one of DEB_SIGN_KEY or DEB_SIGN_KEY_ID or DEB_SIGN_KEY_PATH vars must be defined " >&2
+		exit 1
+	fi
+
+	# Import GPG signing private key if it is provided
+	if [[ -n "$${DEB_SIGN_KEY_ID}" ]]; then
+		# Check if gpg knows about this key id
+		if [[ $$(gpg --list-keys $${DEB_SIGN_KEY_ID} 2>&1) =~ "No public key" ]]; then
+			echo "Error: No public key $${DEB_SIGN_KEY_ID}" >&2
+			exit 1
+		else
+	    	SIGN_ARGS="-k${DEB_SIGN_KEY_ID}"
+		fi
+	elif [[ -n "$${DEB_SIGN_KEY}" ]]; then
+		echo "$${DEB_SIGN_KEY}" | gpg --import
+		KEY_ID=$$(gpg --list-keys --with-colon | awk -F: '/^fpr/ {print $$10;exit}')
+		if [[ -z $${KEY_ID} ]]; then
+			echo "Error: Unable to import signing key from var DEB_SIGN_KEY" >&2
+			exit 1
+		fi
+	    SIGN_ARGS="-k$${KEY_ID}"
+	elif [[ -n "$${DEB_SIGN_KEY_PATH}" ]]; then
+		gpg --import --with-colons "$${DEB_SIGN_KEY_PATH}"
+		KEY_ID=$$(gpg --list-keys --with-colon | awk -F: '/^fpr/ {print $$10;exit}')
+		if [[ -z $${KEY_ID} ]]; then
+			echo "Error: Unable to import signing key from path: $${DEB_SIGN_KEY_PATH}" >&2
+			exit 1
+		fi
+	    SIGN_ARGS="-k$${KEY_ID}"
+	else
+		SIGN_ARGS="-us -uc"
+	fi
+
+	# Build package
+	(cd debian && debuild --preserve-env --check-dirname-level 0 $${SIGN_ARGS})
+
+	# Move debian package and signed metadata files to the output dir
+	DEB_FILES=$$(echo ../$(PROJECT_NAME)*.{deb,dsc,changes,buildinfo})
+	mkdir -p $(BUILD_DEB_OUTPUT_DIR) && mv $$DEB_FILES $(BUILD_DEB_OUTPUT_DIR)
 
 
 .PHONY: clean_debuild

ch_tools/__init__.py

@@ -1,3 +1,3 @@
 """A set of tools for administration and diagnostics of ClickHouse DBMS."""
 
-__version__ = "1.0.0"
+__version__ = "2.548.182050718"

debian/changelog

@@ -1,6 +1,9 @@
-clickhouse-tools (1.0.0) UNRELEASED; urgency=low
+clickhouse-tools (2.548.182050718) stable; urgency=low
 
+  [ Dmitry Starov ]
   * Initial Release.
 
- -- Dmitry Starov <[email protected]>  Thu, 01 Jun 2023 16:00:00 +0300
-
+  [ Yandex LLC ]
+  * Autobuild
+
+ -- Yandex LLC <[email protected]>  Sun, 26 Nov 2023 14:36:53 +0000

debian/clickhouse-tools.substvars

@@ -0,0 +1,2 @@
+misc:Depends=
+misc:Pre-Depends=

debian/debhelper-build-stamp

@@ -0,0 +1 @@
+clickhouse-tools

debian/files

@@ -0,0 +1,2 @@
+clickhouse-tools_2.548.182050718_amd64.buildinfo database optional
+clickhouse-tools_2.548.182050718_amd64.deb database optional

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "clickhouse-tools"
-version = "0.1.0"
+version = "2.548.182050718"
 license = "MIT"
 description = "clickhouse-tools is a set of tools for administration and diagnostics of ClickHouse DBMS."
 
@@ -155,4 +155,4 @@ exclude = ['tests/staging']
 
 [tool.codespell]
 skip = 'poetry.lock'
-ignore-words-list = 'sav'
+ignore-words-list = 'sav,fpr'