diff --git a/.codespellrc b/.codespellrc index 58efa255..fef36321 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,3 +1,3 @@ [codespell] skip = staging,venv* -ignore-words-list = sav +ignore-words-list = sav,fpr diff --git a/.gitignore b/.gitignore index 968a88b9..94d3fffc 100644 --- a/.gitignore +++ b/.gitignore @@ -13,11 +13,13 @@ .session_conf.sav .tox/ .vscode +build/ ch_backup/version.txt debian/ch-backup* debian/changelog debian/files htmlcov/ +out/ staging/ venv/ diff --git a/Dockerfile-deb-build b/Dockerfile-deb-build new file mode 100644 index 00000000..e2a87db3 --- /dev/null +++ b/Dockerfile-deb-build @@ -0,0 +1,42 @@ +ARG BASE_IMAGE=ubuntu:22.04 +FROM --platform=$TARGETPLATFORM $BASE_IMAGE + +ARG DEBIAN_FRONTEND=noninteractive + +RUN set -ex \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # Debian packaging tools + build-essential \ + debhelper \ + devscripts \ + fakeroot \ + # Managing keys for debian package signing + gpg \ + gpg-agent \ + # Python packaging tools + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-venv \ + # Misc + curl \ + locales \ + # For building PyNacl library + libffi-dev libssl-dev libboost-all-dev libsodium-dev \ + # Configure locales + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 \ + # Ensure that `python` refers to `python3` so that poetry works. + # It makes sense for ubuntu:18.04 + && ln -sf /usr/bin/python3 /usr/bin/python + +# Project directory must be mounted here +VOLUME /src +WORKDIR /src + +# For compiling PyNACL library, which is used by ch-backup +# See https://pynacl.readthedocs.io/en/latest/install/#linux-source-build +ENV SODIUM_INSTALL=system + +CMD ["make", "build-deb-package-local"] diff --git a/Makefile b/Makefile index ad518692..c8865d16 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,11 @@ +SHELL := bash + export PYTHON?=python3 export PYTHONIOENCODING?=utf8 export NO_VENV?= export COMPOSE_HTTP_TIMEOUT?=300 export CLICKHOUSE_VERSION?=latest +export PROJECT_NAME ?= ch-backup ifndef NO_VENV PATH:=venv/bin:${PATH} @@ -11,6 +14,8 @@ endif PYTHON_VERSION=$(shell ${PYTHON} -c 'import sys; print(".".join(map(str, sys.version_info[:2])))') SESSION_FILE=.session_conf.sav INSTALL_DIR=$(DESTDIR)/opt/yandex/ch-backup +SRC_DIR ?= ch_backup +TESTS_DIR ?= tests TEST_ENV=env \ PATH=${PATH} \ @@ -21,6 +26,24 @@ TEST_ENV=env \ INTEGRATION_TEST_TOOL=${TEST_ENV} python -m tests.integration.env_control +export BUILD_PYTHON_OUTPUT_DIR ?= dist +export BUILD_DEB_OUTPUT_DIR ?= out + +# Different ways of passing signing key for building debian package +export DEB_SIGN_KEY_ID ?= +export DEB_SIGN_KEY ?= +export DEB_SIGN_KEY_PATH ?= + +# Platform of image for building debian package according to +# https://docs.docker.com/build/building/multi-platform/#building-multi-platform-images +# E.g. linux/amd64, linux/arm64, etc. +# If platform is not provided Docker uses platform of the host performing the build +export DEB_TARGET_PLATFORM ?= +# Name of image (with tag) for building deb package. +# E.g. ubuntu:22.04, ubuntu:jammy, ubuntu:bionic, etc. +# If it is not provided, default value in Dockerfile is used +export DEB_BUILD_DISTRIBUTION ?= + .PHONY: build build: install-deps ch_backup/version.txt @@ -33,11 +56,11 @@ lint: install-deps isort black codespell ruff pylint mypy bandit .PHONY: isort isort: install-deps - ${TEST_ENV} isort --check --diff . + ${TEST_ENV} isort --check --diff $(SRC_DIR) $(TESTS_DIR) .PHONY: black black: install-deps - ${TEST_ENV} black --check --diff . + ${TEST_ENV} black --check --diff $(SRC_DIR) $(TESTS_DIR) .PHONY: codespell codespell: install-deps @@ -49,16 +72,16 @@ fix-codespell-errors: install-deps .PHONY: ruff ruff: install-deps - ${TEST_ENV} ruff check ch_backup tests + ${TEST_ENV} ruff check $(SRC_DIR) $(TESTS_DIR) .PHONY: pylint pylint: install-deps - ${TEST_ENV} pylint ch_backup - ${TEST_ENV} pylint --disable=missing-docstring,invalid-name tests + ${TEST_ENV} pylint $(SRC_DIR) + ${TEST_ENV} pylint --disable=missing-docstring,invalid-name $(TESTS_DIR) .PHONY: mypy mypy: install-deps - ${TEST_ENV} mypy ch_backup tests + ${TEST_ENV} mypy $(SRC_DIR) $(TESTS_DIR) .PHONY: bandit bandit: install-deps @@ -77,7 +100,7 @@ test-integration: build create-env .PHONY: clean -clean: clean-env clean-pycache +clean: clean-env clean-pycache clean-debuild rm -rf venv *.egg-info htmlcov .coverage* .hypothesis .mypy_cache .pytest_cache .install-deps ch_backup/version.txt .PHONY: clean-pycache @@ -111,13 +134,20 @@ uninstall: rm -rf $(INSTALL_DIR) $(DESTDIR)/usr/bin/ch-backup $(DESTDIR)/etc/bash_completion.d/ch-backup -.PHONY: debuild -debuild: debian-changelog - cd debian && \ - debuild --check-dirname-level 0 --no-tgz-check --preserve-env -uc -us -.PHONY: debian-changelog -debian-changelog: build + +.PHONY: build-deb-package +build-deb-package: + ./build_deb_in_docker.sh + + +.PHONY: build-deb-package-local +build-deb-package-local: prepare-changelog + ./build_deb.sh + + +.PHONY: prepare-changelog +prepare-changelog: build @rm -f debian/changelog dch --create --package ch-backup --distribution stable \ -v `cat ch_backup/version.txt` \ @@ -125,8 +155,8 @@ debian-changelog: build .PHONY: clean-debuild -clean-debuild: clean - rm -rf debian/{changelog,files,ch-backup*} +clean-debuild: + rm -rf debian/{changelog,files,ch-backup,.debhelper} rm -f ../ch-backup_*{build,changes,deb,dsc,tar.gz} diff --git a/build_deb.sh b/build_deb.sh new file mode 100755 index 00000000..4df85a6e --- /dev/null +++ b/build_deb.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -e + +# Sanitize package signing options +COUNT=0 +for sign_param in DEB_SIGN_KEY DEB_SIGN_KEY_ID DEB_SIGN_KEY_PATH; do + if [[ -n "${!sign_param}" ]]; then ((COUNT+=1)); fi +done +if (( COUNT > 1 )); then + echo "Error: At most one of DEB_SIGN_KEY or DEB_SIGN_KEY_ID or DEB_SIGN_KEY_PATH vars must be defined " >&2 + exit 1 +fi + +# Import GPG signing private key if it is provided +if [[ -n "${DEB_SIGN_KEY_ID}" ]]; then + # Check if gpg knows about this key id + if [[ $(gpg --list-keys ${DEB_SIGN_KEY_ID} 2>&1) =~ "No public key" ]]; then + echo "Error: No public key ${DEB_SIGN_KEY_ID}" >&2 + exit 1 + else + SIGN_ARGS="-k${DEB_SIGN_KEY_ID}" + fi +elif [[ -n "${DEB_SIGN_KEY}" ]]; then + echo "${DEB_SIGN_KEY}" | gpg --import + KEY_ID=$(gpg --list-keys --with-colon | awk -F: '/^fpr/ {print $10;exit}') + if [[ -z ${KEY_ID} ]]; then + echo "Error: Unable to import signing key from var DEB_SIGN_KEY" >&2 + exit 1 + fi + SIGN_ARGS="-k${KEY_ID}" +elif [[ -n "${DEB_SIGN_KEY_PATH}" ]]; then + gpg --import --with-colons "${DEB_SIGN_KEY_PATH}" + KEY_ID=$(gpg --list-keys --with-colon | awk -F: '/^fpr/ {print $10;exit}') + if [[ -z ${KEY_ID} ]]; then + echo "Error: Unable to import signing key from path: ${DEB_SIGN_KEY_PATH}" >&2 + exit 1 + fi + SIGN_ARGS="-k${KEY_ID}" +else + # Do not sign debian package + SIGN_ARGS="-us -uc" +fi + +# Build package +(cd debian && debuild --preserve-env --check-dirname-level 0 ${SIGN_ARGS}) + +# Move debian package and signed metadata files to the output dir +DEB_FILES=$(echo ../${PROJECT_NAME}*.{deb,dsc,changes,buildinfo,tar.*}) +mkdir -p ${BUILD_DEB_OUTPUT_DIR} && mv $DEB_FILES ${BUILD_DEB_OUTPUT_DIR} diff --git a/build_deb_in_docker.sh b/build_deb_in_docker.sh new file mode 100755 index 00000000..790f3e8d --- /dev/null +++ b/build_deb_in_docker.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -e + +BUILD_IMAGE=${PROJECT_NAME}-build +BUILD_ARGS=() + +# Compose image name and build arguments +# Example of image name "clickhouse-tools-build-linux-amd64-linux-bionic" +if [[ -n "${DEB_TARGET_PLATFORM}" ]]; then + BUILD_ARGS+=(--platform=${DEB_TARGET_PLATFORM}) + BUILD_IMAGE="${BUILD_IMAGE}-${DEB_TARGET_PLATFORM}" +fi +if [[ -n "${DEB_BUILD_DISTRIBUTION}" ]]; then + BUILD_ARGS+=(--build-arg BASE_IMAGE=${DEB_BUILD_DISTRIBUTION}) + BUILD_IMAGE="${BUILD_IMAGE}-${DEB_BUILD_DISTRIBUTION}" +fi +# Normalize docker image name +BUILD_IMAGE=$(echo ${BUILD_IMAGE} | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g') + +RUN_ARGS=( \ + -v ${PWD}:/src \ + --env BUILD_DEB_OUTPUT_DIR="${BUILD_DEB_OUTPUT_DIR}" \ + --env DEB_SIGN_KEY="${DEB_SIGN_KEY}" \ + --env DEB_SIGN_KEY_ID="${DEB_SIGN_KEY_ID}" \ +) +# Mount signing key file if its path is provided +if [[ -n "${DEB_SIGN_KEY_PATH}" ]]; then + RUN_ARGS+=( \ + -v ${DEB_SIGN_KEY_PATH}:/signing_key \ + --env DEB_SIGN_KEY_PATH=/signing_key \ + ) +fi + +docker build "${BUILD_ARGS[@]}" -t "${BUILD_IMAGE}" -f Dockerfile-deb-build . +docker run "${RUN_ARGS[@]}" "${BUILD_IMAGE}"