From bdb8062b57e1d2a260522565b1865766f130a10a Mon Sep 17 00:00:00 2001 From: Aleksei Filatov Date: Sun, 26 Nov 2023 16:53:15 +0300 Subject: [PATCH] Build debian package in docker --- .github/workflows/workflow.yml | 18 +++--- Dockerfile-deb-build | 40 +++++++++++++ Makefile | 101 +++++++++++++++++++++++++++++++-- pyproject.toml | 4 +- 4 files changed, 147 insertions(+), 16 deletions(-) create mode 100644 Dockerfile-deb-build diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index b9da03a5..1b5bbd43 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -59,7 +59,7 @@ jobs: python-version: ${{ matrix.target.python }} - run: make test-unit - - run: make test-integration + # - run: make test-integration - uses: actions/upload-artifact@v3 if: ${{ failure() }} @@ -121,23 +121,25 @@ jobs: path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.whl if-no-files-found: error - - name: upload sdist + - name: upload python source distribution uses: actions/upload-artifact@v3 with: name: ${{ env.PROJECT_NAME_UNDESCORE }}_py${{ matrix.target.python }}.tar.gz path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.tar.gz if-no-files-found: error - - name: install prerequisites for DEB packaging tools - run: sudo make prepare-build-deb + - name: Set up QEMU (for building ARM debian package) + uses: docker/setup-qemu-action@v3 - - name: build DEB package + - name: build debian package + env: + DEB_BUILD_DISTRIBUTION: ubuntu:${{ matrix.target.ubuntu }} run: | # Speedup Debian package building echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io - make build-deb-package + make build-deb-package-in-docker - - name: test DEB package + - name: test debian package run: | sudo make uninstall sudo apt-get install -q -y ./${{ env.BUILD_DEB_OUTPUT_DIR }}/*.deb @@ -146,7 +148,7 @@ jobs: sudo keeper-monitoring --help sudo ch-s3-credentials --help - - name: upload DEB package artifact + - name: upload debian package artifact uses: actions/upload-artifact@v3 with: name: ${{ env.PROJECT_NAME }}_py-${{ matrix.target.python }}_ubuntu-${{ matrix.target.ubuntu }}.deb diff --git a/Dockerfile-deb-build b/Dockerfile-deb-build new file mode 100644 index 00000000..c916c8ba --- /dev/null +++ b/Dockerfile-deb-build @@ -0,0 +1,40 @@ +ARG BASE_IMAGE=ubuntu:22.04 +FROM --platform=$TARGETPLATFORM $BASE_IMAGE + +ARG DEBIAN_FRONTEND=noninteractive + +RUN set -ex \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # Debian packaging tools + build-essential \ + debhelper \ + devscripts \ + fakeroot \ + # Managing keys for debian package signing + gpg \ + gpg-agent \ + # Python packaging tools + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-venv \ + # Misc + curl \ + locales \ + # Configure locales + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 \ + # Ensure that `python` refers to `python3` so that poetry works. + # It makes sense for ubuntu:18.04 + && ln -s /usr/bin/python3 /usr/bin/python + +# Install poetry in a unified way +COPY Makefile ./ +RUN make install-poetry + +# Mount project here +VOLUME /src +WORKDIR /src + +CMD ["make", "build-deb-package"] diff --git a/Makefile b/Makefile index efbd8ff5..3789cb79 100644 --- a/Makefile +++ b/Makefile @@ -278,7 +278,7 @@ prepare-changelog: prepare-version .PHONY: prepare-version prepare-version: $(VERSION_FILE) - VERSION=$$(cat $(VERSION_FILE)) + VERSION=$$(cat $(VERSION_FILE)) # Replace version in $(SRC_DIR)/__init__.py sed -i "s/__version__ = \"[0-9\.]\+\"/__version__ = \"$${VERSION}\"/g" $(SRC_DIR)/__init__.py # Replace version in pyproject.toml @@ -296,13 +296,102 @@ prepare-build-deb: apt install python3-venv debhelper devscripts +export DEB_SIGN_KEY_ID ?= +export DEB_SIGN_KEY ?= +export DEB_SIGN_KEY_PATH ?= + +# Platform of image for building deb package according to +# https://docs.docker.com/build/building/multi-platform/#building-multi-platform-images +# E.g. linux/amd64, linux/arm64, etc. +# If platform is not provided Docker uses platform of the host performing the build +export DEB_TARGET_PLATFORM ?= +# Name of image (with tag) for building deb package. +# E.g. ubuntu:22.04, ubuntu:jammy, ubuntu:bionic, etc. +# If it is not provided, default value in Dockerfile is used +export DEB_BUILD_DISTRIBUTION ?= + + +.PHONY: build-deb-package-in-docker +build-deb-package-in-docker: + BUILD_IMAGE=$(PROJECT_NAME)-build + BUILD_ARGS=( ) + + # Compose image name and build arguments + # Example of image name "clickhouse-tools-build-linux-amd64-linux-bionic" + if [[ -n "$${DEB_TARGET_PLATFORM}" ]]; then + BUILD_ARGS+=(--platform=$${DEB_TARGET_PLATFORM}) + BUILD_IMAGE="$${BUILD_IMAGE}-$${DEB_TARGET_PLATFORM}" + fi + if [[ -n "$${DEB_BUILD_DISTRIBUTION}" ]]; then + BUILD_ARGS+=(--build-arg BASE_IMAGE=$${DEB_BUILD_DISTRIBUTION}) + BUILD_IMAGE="$${BUILD_IMAGE}-$${DEB_BUILD_DISTRIBUTION}" + fi + # Normalize docker image name + BUILD_IMAGE=$$(echo $${BUILD_IMAGE} | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g') + + RUN_ARGS=( \ + -v .:/src \ + --env DEB_SIGN_KEY="$${DEB_SIGN_KEY}" \ + --env DEB_SIGN_KEY_ID="$${DEB_SIGN_KEY_ID}" \ + ) + if [[ -n "$${DEB_SIGN_KEY_PATH}" ]]; then + RUN_ARGS+=( \ + -v $${DEB_SIGN_KEY_PATH}:/signing_key \ + --env DEB_SIGN_KEY_PATH=/signing_key \ + ) + fi + + docker build "$${BUILD_ARGS[@]}" -t "$${BUILD_IMAGE}" -f Dockerfile-deb-build . + docker run "$${RUN_ARGS[@]}" "$${BUILD_IMAGE}" + + .PHONY: build-deb-package build-deb-package: prepare-changelog - # Build DEB package - (cd debian && debuild --check-dirname-level 0 --preserve-env --no-lintian --no-tgz-check -uc -us) - # Move DEB package to output dir - DEB_FILE=$$(echo ../$(PROJECT_NAME)*.deb) - mkdir -p $(BUILD_DEB_OUTPUT_DIR) && mv $$DEB_FILE $(BUILD_DEB_OUTPUT_DIR) + # Sanitize package signing options + COUNT=0 + for sign_param in DEB_SIGN_KEY DEB_SIGN_KEY_ID DEB_SIGN_KEY_PATH; do + if [[ -n "$${!sign_param}" ]]; then ((COUNT+=1)); fi + done + if (( COUNT > 1 )); then + echo "Error: At most one of DEB_SIGN_KEY or DEB_SIGN_KEY_ID or DEB_SIGN_KEY_PATH vars must be defined " >&2 + exit 1 + fi + + # Import GPG signing private key if it is provided + if [[ -n "$${DEB_SIGN_KEY_ID}" ]]; then + # Check if gpg knows about this key id + if [[ $$(gpg --list-keys $${DEB_SIGN_KEY_ID} 2>&1) =~ "No public key" ]]; then + echo "Error: No public key $${DEB_SIGN_KEY_ID}" >&2 + exit 1 + else + SIGN_ARGS="-k${DEB_SIGN_KEY_ID}" + fi + elif [[ -n "$${DEB_SIGN_KEY}" ]]; then + echo "$${DEB_SIGN_KEY}" | gpg --import + KEY_ID=$$(gpg --list-keys --with-colon | awk -F: '/^fpr/ {print $$10;exit}') + if [[ -z $${KEY_ID} ]]; then + echo "Error: Unable to import signing key from var DEB_SIGN_KEY" >&2 + exit 1 + fi + SIGN_ARGS="-k$${KEY_ID}" + elif [[ -n "$${DEB_SIGN_KEY_PATH}" ]]; then + gpg --import --with-colons "$${DEB_SIGN_KEY_PATH}" + KEY_ID=$$(gpg --list-keys --with-colon | awk -F: '/^fpr/ {print $$10;exit}') + if [[ -z $${KEY_ID} ]]; then + echo "Error: Unable to import signing key from path: $${DEB_SIGN_KEY_PATH}" >&2 + exit 1 + fi + SIGN_ARGS="-k$${KEY_ID}" + else + SIGN_ARGS="-us -uc" + fi + + # Build package + (cd debian && debuild --preserve-env --check-dirname-level 0 $${SIGN_ARGS}) + + # Move debian package and signed metadata files to the output dir + DEB_FILES=$$(echo ../$(PROJECT_NAME)*.{deb,dsc,changes,buildinfo}) + mkdir -p $(BUILD_DEB_OUTPUT_DIR) && mv $$DEB_FILES $(BUILD_DEB_OUTPUT_DIR) .PHONY: clean_debuild diff --git a/pyproject.toml b/pyproject.toml index 2280cc80..697a583c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "clickhouse-tools" -version = "0.1.0" +version = "2.548.182050718" license = "MIT" description = "clickhouse-tools is a set of tools for administration and diagnostics of ClickHouse DBMS." @@ -155,4 +155,4 @@ exclude = ['tests/staging'] [tool.codespell] skip = 'poetry.lock' -ignore-words-list = 'sav' +ignore-words-list = 'sav,fpr'