Build debian package in docker

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
 skip = staging,venv*
-ignore-words-list = sav
+ignore-words-list = sav,fpr

.gitignore

@@ -13,11 +13,13 @@
 .session_conf.sav
 .tox/
 .vscode
+build/
 ch_backup/version.txt
 debian/ch-backup*
 debian/changelog
 debian/files
 htmlcov/
+out/
 staging/
 venv/
 

Dockerfile-deb-build

@@ -0,0 +1,42 @@
+ARG BASE_IMAGE=ubuntu:22.04
+FROM --platform=$TARGETPLATFORM $BASE_IMAGE
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        # Debian packaging tools
+        build-essential \
+        debhelper \
+        devscripts \
+        fakeroot \
+        # Managing keys for debian package signing
+        gpg \
+        gpg-agent \
+        # Python packaging tools
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-venv \
+        # Misc
+        curl \
+        locales \
+        # For building PyNacl library
+        libffi-dev libssl-dev libboost-all-dev libsodium-dev \
+    # Configure locales
+    && locale-gen en_US.UTF-8 \
+    && update-locale LANG=en_US.UTF-8 \
+    # Ensure that `python` refers to `python3` so that poetry works. 
+    # It makes sense for ubuntu:18.04
+    && ln -sf /usr/bin/python3 /usr/bin/python
+
+# Project directory must be mounted here
+VOLUME /src
+WORKDIR /src
+
+# For compiling PyNACL library, which is used by ch-backup
+# See https://pynacl.readthedocs.io/en/latest/install/#linux-source-build
+ENV SODIUM_INSTALL=system
+
+CMD ["make", "build-deb-package-local"]

Makefile

@@ -1,8 +1,11 @@
+SHELL := bash
+
 export PYTHON?=python3
 export PYTHONIOENCODING?=utf8
 export NO_VENV?=
 export COMPOSE_HTTP_TIMEOUT?=300
 export CLICKHOUSE_VERSION?=latest
+export PROJECT_NAME ?= ch-backup
 
 ifndef NO_VENV
   PATH:=venv/bin:${PATH}
@@ -11,6 +14,8 @@ endif
 PYTHON_VERSION=$(shell ${PYTHON} -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')
 SESSION_FILE=.session_conf.sav
 INSTALL_DIR=$(DESTDIR)/opt/yandex/ch-backup
+SRC_DIR ?= ch_backup
+TESTS_DIR ?= tests
 
 TEST_ENV=env \
     PATH=${PATH} \
@@ -21,6 +26,24 @@ TEST_ENV=env \
 
 INTEGRATION_TEST_TOOL=${TEST_ENV} python -m tests.integration.env_control
 
+export BUILD_PYTHON_OUTPUT_DIR ?= dist
+export BUILD_DEB_OUTPUT_DIR ?= out
+
+# Different ways of passing signing key for building debian package
+export DEB_SIGN_KEY_ID ?=
+export DEB_SIGN_KEY ?=
+export DEB_SIGN_KEY_PATH ?=
+
+# Platform of image for building debian package according to
+# https://docs.docker.com/build/building/multi-platform/#building-multi-platform-images
+# E.g. linux/amd64, linux/arm64, etc.
+# If platform is not provided Docker uses platform of the host performing the build
+export DEB_TARGET_PLATFORM ?=
+# Name of image (with tag) for building deb package.
+# E.g. ubuntu:22.04, ubuntu:jammy, ubuntu:bionic, etc.
+# If it is not provided, default value in Dockerfile is used
+export DEB_BUILD_DISTRIBUTION ?=
+
 
 .PHONY: build
 build: install-deps ch_backup/version.txt
@@ -33,11 +56,11 @@ lint: install-deps isort black codespell ruff pylint mypy bandit
 
 .PHONY: isort
 isort: install-deps
-	${TEST_ENV} isort --check --diff .
+	${TEST_ENV} isort --check --diff $(SRC_DIR) $(TESTS_DIR)
 
 .PHONY: black
 black: install-deps
-	${TEST_ENV} black --check --diff .
+	${TEST_ENV} black --check --diff $(SRC_DIR) $(TESTS_DIR)
 
 .PHONY: codespell
 codespell: install-deps
@@ -49,16 +72,16 @@ fix-codespell-errors: install-deps
 
 .PHONY: ruff
 ruff: install-deps
-	${TEST_ENV} ruff check ch_backup tests
+	${TEST_ENV} ruff check $(SRC_DIR) $(TESTS_DIR)
 
 .PHONY: pylint
 pylint: install-deps
-	${TEST_ENV} pylint ch_backup
-	${TEST_ENV} pylint --disable=missing-docstring,invalid-name tests
+	${TEST_ENV} pylint $(SRC_DIR)
+	${TEST_ENV} pylint --disable=missing-docstring,invalid-name $(TESTS_DIR)
 
 .PHONY: mypy
 mypy: install-deps
-	${TEST_ENV} mypy ch_backup tests
+	${TEST_ENV} mypy $(SRC_DIR) $(TESTS_DIR)
 
 .PHONY: bandit
 bandit: install-deps
@@ -77,7 +100,7 @@ test-integration: build create-env
 
 
 .PHONY: clean
-clean: clean-env clean-pycache
+clean: clean-env clean-pycache clean-debuild
 	rm -rf venv *.egg-info htmlcov .coverage* .hypothesis .mypy_cache .pytest_cache .install-deps ch_backup/version.txt
 
 .PHONY: clean-pycache
@@ -111,22 +134,29 @@ uninstall:
 	rm -rf $(INSTALL_DIR) $(DESTDIR)/usr/bin/ch-backup $(DESTDIR)/etc/bash_completion.d/ch-backup
 
 
-.PHONY: debuild
-debuild: debian-changelog
-	cd debian && \
-	    debuild --check-dirname-level 0 --no-tgz-check --preserve-env -uc -us
 
-.PHONY: debian-changelog
-debian-changelog: build
+
+.PHONY: build-deb-package
+build-deb-package:
+	./build_deb_in_docker.sh
+
+
+.PHONY: build-deb-package-local
+build-deb-package-local: prepare-changelog
+	./build_deb.sh
+
+
+.PHONY: prepare-changelog
+prepare-changelog: build
 	@rm -f debian/changelog
 	dch --create --package ch-backup --distribution stable \
 	    -v `cat ch_backup/version.txt` \
 	    "Yandex autobuild"
 
 
 .PHONY: clean-debuild
-clean-debuild: clean
-	rm -rf debian/{changelog,files,ch-backup*}
+clean-debuild:
+	rm -rf debian/{changelog,files,ch-backup,.debhelper}
 	rm -f ../ch-backup_*{build,changes,deb,dsc,tar.gz}
 
 

build_deb.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Sanitize package signing options
+COUNT=0
+for sign_param in DEB_SIGN_KEY DEB_SIGN_KEY_ID DEB_SIGN_KEY_PATH; do
+    if [[ -n "${!sign_param}" ]]; then ((COUNT+=1)); fi
+done
+if (( COUNT > 1 )); then
+    echo "Error: At most one of DEB_SIGN_KEY or DEB_SIGN_KEY_ID or DEB_SIGN_KEY_PATH vars must be defined " >&2
+    exit 1
+fi
+
+# Import GPG signing private key if it is provided
+if [[ -n "${DEB_SIGN_KEY_ID}" ]]; then
+    # Check if gpg knows about this key id
+    if [[ $(gpg --list-keys ${DEB_SIGN_KEY_ID} 2>&1) =~ "No public key" ]]; then
+        echo "Error: No public key ${DEB_SIGN_KEY_ID}" >&2
+        exit 1
+    else
+        SIGN_ARGS="-k${DEB_SIGN_KEY_ID}"
+    fi
+elif [[ -n "${DEB_SIGN_KEY}" ]]; then
+    echo "${DEB_SIGN_KEY}" | gpg --import
+    KEY_ID=$(gpg --list-keys --with-colon | awk -F: '/^fpr/ {print $10;exit}')
+    if [[ -z ${KEY_ID} ]]; then
+        echo "Error: Unable to import signing key from var DEB_SIGN_KEY" >&2
+        exit 1
+    fi
+    SIGN_ARGS="-k${KEY_ID}"
+elif [[ -n "${DEB_SIGN_KEY_PATH}" ]]; then
+    gpg --import --with-colons "${DEB_SIGN_KEY_PATH}"
+    KEY_ID=$(gpg --list-keys --with-colon | awk -F: '/^fpr/ {print $10;exit}')
+    if [[ -z ${KEY_ID} ]]; then
+        echo "Error: Unable to import signing key from path: ${DEB_SIGN_KEY_PATH}" >&2
+        exit 1
+    fi
+    SIGN_ARGS="-k${KEY_ID}"
+else
+    # Do not sign debian package
+    SIGN_ARGS="-us -uc"
+fi
+
+# Build package
+(cd debian && debuild --preserve-env --check-dirname-level 0 ${SIGN_ARGS})
+
+# Move debian package and signed metadata files to the output dir
+DEB_FILES=$(echo ../${PROJECT_NAME}*.{deb,dsc,changes,buildinfo,tar.*})
+mkdir -p ${BUILD_DEB_OUTPUT_DIR} && mv $DEB_FILES ${BUILD_DEB_OUTPUT_DIR}

build_deb_in_docker.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -e
+
+BUILD_IMAGE=${PROJECT_NAME}-build
+BUILD_ARGS=()
+
+# Compose image name and build arguments
+# Example of image name "clickhouse-tools-build-linux-amd64-linux-bionic"		
+if [[ -n "${DEB_TARGET_PLATFORM}" ]]; then
+    BUILD_ARGS+=(--platform=${DEB_TARGET_PLATFORM})
+    BUILD_IMAGE="${BUILD_IMAGE}-${DEB_TARGET_PLATFORM}"
+fi
+if [[ -n "${DEB_BUILD_DISTRIBUTION}" ]]; then
+    BUILD_ARGS+=(--build-arg BASE_IMAGE=${DEB_BUILD_DISTRIBUTION})
+    BUILD_IMAGE="${BUILD_IMAGE}-${DEB_BUILD_DISTRIBUTION}"
+fi
+# Normalize docker image name
+BUILD_IMAGE=$(echo ${BUILD_IMAGE} | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g')
+
+RUN_ARGS=( \
+    -v ${PWD}:/src \
+    --env BUILD_DEB_OUTPUT_DIR="${BUILD_DEB_OUTPUT_DIR}" \
+    --env DEB_SIGN_KEY="${DEB_SIGN_KEY}" \
+    --env DEB_SIGN_KEY_ID="${DEB_SIGN_KEY_ID}" \
+)
+# Mount signing key file if its path is provided
+if [[ -n "${DEB_SIGN_KEY_PATH}" ]]; then
+    RUN_ARGS+=( \
+        -v ${DEB_SIGN_KEY_PATH}:/signing_key \
+        --env DEB_SIGN_KEY_PATH=/signing_key \
+    )
+fi
+
+docker build "${BUILD_ARGS[@]}" -t "${BUILD_IMAGE}" -f Dockerfile-deb-build .
+docker run "${RUN_ARGS[@]}" "${BUILD_IMAGE}"