Skip to content

Commit

Permalink
Build debian package in docker
Browse files Browse the repository at this point in the history
  • Loading branch information
aalexfvk committed Nov 26, 2023
1 parent b3c1b89 commit bdb8062
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 16 deletions.
18 changes: 10 additions & 8 deletions .github/workflows/workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:
python-version: ${{ matrix.target.python }}

- run: make test-unit
- run: make test-integration
# - run: make test-integration

- uses: actions/upload-artifact@v3
if: ${{ failure() }}
Expand Down Expand Up @@ -121,23 +121,25 @@ jobs:
path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.whl
if-no-files-found: error

- name: upload sdist
- name: upload python source distribution
uses: actions/upload-artifact@v3
with:
name: ${{ env.PROJECT_NAME_UNDESCORE }}_py${{ matrix.target.python }}.tar.gz
path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.tar.gz
if-no-files-found: error

- name: install prerequisites for DEB packaging tools
run: sudo make prepare-build-deb
- name: Set up QEMU (for building ARM debian package)
uses: docker/setup-qemu-action@v3

- name: build DEB package
- name: build debian package
env:
DEB_BUILD_DISTRIBUTION: ubuntu:${{ matrix.target.ubuntu }}
run: |
# Speedup Debian package building
echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io
make build-deb-package
make build-deb-package-in-docker
- name: test DEB package
- name: test debian package
run: |
sudo make uninstall
sudo apt-get install -q -y ./${{ env.BUILD_DEB_OUTPUT_DIR }}/*.deb
Expand All @@ -146,7 +148,7 @@ jobs:
sudo keeper-monitoring --help
sudo ch-s3-credentials --help
- name: upload DEB package artifact
- name: upload debian package artifact
uses: actions/upload-artifact@v3
with:
name: ${{ env.PROJECT_NAME }}_py-${{ matrix.target.python }}_ubuntu-${{ matrix.target.ubuntu }}.deb
Expand Down
40 changes: 40 additions & 0 deletions Dockerfile-deb-build
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
ARG BASE_IMAGE=ubuntu:22.04
FROM --platform=$TARGETPLATFORM $BASE_IMAGE

ARG DEBIAN_FRONTEND=noninteractive

RUN set -ex \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
# Debian packaging tools
build-essential \
debhelper \
devscripts \
fakeroot \
# Managing keys for debian package signing
gpg \
gpg-agent \
# Python packaging tools
python3-dev \
python3-pip \
python3-setuptools \
python3-venv \
# Misc
curl \
locales \
# Configure locales
&& locale-gen en_US.UTF-8 \
&& update-locale LANG=en_US.UTF-8 \
# Ensure that `python` refers to `python3` so that poetry works.
# It makes sense for ubuntu:18.04
&& ln -s /usr/bin/python3 /usr/bin/python

# Install poetry in a unified way
COPY Makefile ./
RUN make install-poetry

# Mount project here
VOLUME /src
WORKDIR /src

CMD ["make", "build-deb-package"]
101 changes: 95 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ prepare-changelog: prepare-version

.PHONY: prepare-version
prepare-version: $(VERSION_FILE)
VERSION=$$(cat $(VERSION_FILE))
VERSION=$$(cat $(VERSION_FILE))
# Replace version in $(SRC_DIR)/__init__.py
sed -i "s/__version__ = \"[0-9\.]\+\"/__version__ = \"$${VERSION}\"/g" $(SRC_DIR)/__init__.py
# Replace version in pyproject.toml
Expand All @@ -296,13 +296,102 @@ prepare-build-deb:
apt install python3-venv debhelper devscripts


export DEB_SIGN_KEY_ID ?=
export DEB_SIGN_KEY ?=
export DEB_SIGN_KEY_PATH ?=

# Platform of image for building deb package according to
# https://docs.docker.com/build/building/multi-platform/#building-multi-platform-images
# E.g. linux/amd64, linux/arm64, etc.
# If platform is not provided Docker uses platform of the host performing the build
export DEB_TARGET_PLATFORM ?=
# Name of image (with tag) for building deb package.
# E.g. ubuntu:22.04, ubuntu:jammy, ubuntu:bionic, etc.
# If it is not provided, default value in Dockerfile is used
export DEB_BUILD_DISTRIBUTION ?=


.PHONY: build-deb-package-in-docker
build-deb-package-in-docker:
BUILD_IMAGE=$(PROJECT_NAME)-build
BUILD_ARGS=( )

# Compose image name and build arguments
# Example of image name "clickhouse-tools-build-linux-amd64-linux-bionic"
if [[ -n "$${DEB_TARGET_PLATFORM}" ]]; then
BUILD_ARGS+=(--platform=$${DEB_TARGET_PLATFORM})
BUILD_IMAGE="$${BUILD_IMAGE}-$${DEB_TARGET_PLATFORM}"
fi
if [[ -n "$${DEB_BUILD_DISTRIBUTION}" ]]; then
BUILD_ARGS+=(--build-arg BASE_IMAGE=$${DEB_BUILD_DISTRIBUTION})
BUILD_IMAGE="$${BUILD_IMAGE}-$${DEB_BUILD_DISTRIBUTION}"
fi
# Normalize docker image name
BUILD_IMAGE=$$(echo $${BUILD_IMAGE} | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g')

RUN_ARGS=( \
-v .:/src \
--env DEB_SIGN_KEY="$${DEB_SIGN_KEY}" \
--env DEB_SIGN_KEY_ID="$${DEB_SIGN_KEY_ID}" \
)
if [[ -n "$${DEB_SIGN_KEY_PATH}" ]]; then
RUN_ARGS+=( \
-v $${DEB_SIGN_KEY_PATH}:/signing_key \
--env DEB_SIGN_KEY_PATH=/signing_key \
)
fi

docker build "$${BUILD_ARGS[@]}" -t "$${BUILD_IMAGE}" -f Dockerfile-deb-build .
docker run "$${RUN_ARGS[@]}" "$${BUILD_IMAGE}"


.PHONY: build-deb-package
build-deb-package: prepare-changelog
# Build DEB package
(cd debian && debuild --check-dirname-level 0 --preserve-env --no-lintian --no-tgz-check -uc -us)
# Move DEB package to output dir
DEB_FILE=$$(echo ../$(PROJECT_NAME)*.deb)
mkdir -p $(BUILD_DEB_OUTPUT_DIR) && mv $$DEB_FILE $(BUILD_DEB_OUTPUT_DIR)
# Sanitize package signing options
COUNT=0
for sign_param in DEB_SIGN_KEY DEB_SIGN_KEY_ID DEB_SIGN_KEY_PATH; do
if [[ -n "$${!sign_param}" ]]; then ((COUNT+=1)); fi
done
if (( COUNT > 1 )); then
echo "Error: At most one of DEB_SIGN_KEY or DEB_SIGN_KEY_ID or DEB_SIGN_KEY_PATH vars must be defined " >&2
exit 1
fi

# Import GPG signing private key if it is provided
if [[ -n "$${DEB_SIGN_KEY_ID}" ]]; then
# Check if gpg knows about this key id
if [[ $$(gpg --list-keys $${DEB_SIGN_KEY_ID} 2>&1) =~ "No public key" ]]; then
echo "Error: No public key $${DEB_SIGN_KEY_ID}" >&2
exit 1
else
SIGN_ARGS="-k${DEB_SIGN_KEY_ID}"
fi
elif [[ -n "$${DEB_SIGN_KEY}" ]]; then
echo "$${DEB_SIGN_KEY}" | gpg --import
KEY_ID=$$(gpg --list-keys --with-colon | awk -F: '/^fpr/ {print $$10;exit}')
if [[ -z $${KEY_ID} ]]; then
echo "Error: Unable to import signing key from var DEB_SIGN_KEY" >&2
exit 1
fi
SIGN_ARGS="-k$${KEY_ID}"
elif [[ -n "$${DEB_SIGN_KEY_PATH}" ]]; then
gpg --import --with-colons "$${DEB_SIGN_KEY_PATH}"
KEY_ID=$$(gpg --list-keys --with-colon | awk -F: '/^fpr/ {print $$10;exit}')
if [[ -z $${KEY_ID} ]]; then
echo "Error: Unable to import signing key from path: $${DEB_SIGN_KEY_PATH}" >&2
exit 1
fi
SIGN_ARGS="-k$${KEY_ID}"
else
SIGN_ARGS="-us -uc"
fi

# Build package
(cd debian && debuild --preserve-env --check-dirname-level 0 $${SIGN_ARGS})

# Move debian package and signed metadata files to the output dir
DEB_FILES=$$(echo ../$(PROJECT_NAME)*.{deb,dsc,changes,buildinfo})
mkdir -p $(BUILD_DEB_OUTPUT_DIR) && mv $$DEB_FILES $(BUILD_DEB_OUTPUT_DIR)


.PHONY: clean_debuild
Expand Down
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"

[tool.poetry]
name = "clickhouse-tools"
version = "0.1.0"
version = "2.548.182050718"
license = "MIT"
description = "clickhouse-tools is a set of tools for administration and diagnostics of ClickHouse DBMS."

Expand Down Expand Up @@ -155,4 +155,4 @@ exclude = ['tests/staging']

[tool.codespell]
skip = 'poetry.lock'
ignore-words-list = 'sav'
ignore-words-list = 'sav,fpr'

0 comments on commit bdb8062

Please sign in to comment.