From 18b670646dd2c21a1872b3aca29d70c4f16be65f Mon Sep 17 00:00:00 2001 From: Marc Khouzam Date: Thu, 9 Jan 2025 14:54:04 -0500 Subject: [PATCH] Allow building multiple RPM packages (#837) This allows to have different packages signed with different keys. To achieve this, the building of RPM packages is now done by hack/rpm/build_package.sh while building the final repository is now done by hack/rpm/build_package_repo.sh. This approach allows to call hack/rpm/build_package.sh more than once with different RPM_PACKAGE_NAME and then build the repo with all the new packages as a last step. Signed-off-by: Marc Khouzam --- Makefile | 21 +++++++-- hack/rpm/README.md | 17 +++++-- hack/rpm/build_package.sh | 77 +++++++++--------------------- hack/rpm/build_package_repo.sh | 85 ++++++++++++++++++++++++++++++++++ hack/rpm/tanzu-cli.spec | 11 +---- 5 files changed, 137 insertions(+), 74 deletions(-) create mode 100755 hack/rpm/build_package_repo.sh diff --git a/Makefile b/Makefile index 714a346bf..6201e06bd 100644 --- a/Makefile +++ b/Makefile @@ -189,18 +189,33 @@ apt-package-in-docker: ## Build a debian package from within a container already .PHONY: apt-package apt-package: apt-package-only apt-package-repo ## Build a debian package to use with APT -.PHONY: rpm-package -rpm-package: ## Build an RPM package +.PHONY: rpm-package-only +rpm-package-only: ## Build an RPM package @if [ "$$(command -v docker)" == "" ]; then \ echo "Docker required to build rpm package" ;\ exit 1 ;\ fi - docker run --rm -e VERSION=$(BUILD_VERSION) -e RPM_SIGNER=$(RPM_SIGNER) -e RPM_METADATA_BASE_URI=$(RPM_METADATA_BASE_URI) -v $(ROOT_DIR):$(ROOT_DIR) $(RPM_IMAGE) $(ROOT_DIR)/hack/rpm/build_package.sh + docker run --rm -e VERSION=$(BUILD_VERSION) -e RPM_PACKAGE_NAME=$(RPM_PACKAGE_NAME) -e RPM_SIGNER=$(RPM_SIGNER) -v $(ROOT_DIR):$(ROOT_DIR) $(RPM_IMAGE) $(ROOT_DIR)/hack/rpm/build_package.sh + +.PHONY: rpm-package-repo +rpm-package-repo: ## Build an RPM package repository + @if [ "$$(command -v docker)" == "" ]; then \ + echo "Docker required to build rpm package" ;\ + exit 1 ;\ + fi + docker run --rm -e VERSION=$(BUILD_VERSION) -e RPM_SIGNER=$(RPM_SIGNER) -e RPM_METADATA_BASE_URI=$(RPM_METADATA_BASE_URI) -v $(ROOT_DIR):$(ROOT_DIR) $(RPM_IMAGE) $(ROOT_DIR)/hack/rpm/build_package_repo.sh .PHONY: rpm-package-in-docker rpm-package-in-docker: ## Build an RPM package from within a container already VERSION=$(BUILD_VERSION) $(ROOT_DIR)/hack/rpm/build_package.sh +.PHONY: rpm-package-repo-in-docker +rpm-package-repo-in-docker: ## Build an RPM package repository from within a container already + VERSION=$(BUILD_VERSION) $(ROOT_DIR)/hack/rpm/build_package_repo.sh + +.PHONY: rpm-package +rpm-package: rpm-package-only rpm-package-repo ## Build an RPM package and repository to use with YUM/DNF + .PHONY: choco-package choco-package: ## Build a Chocolatey package @if [ "$$(command -v docker)" = "" ]; then \ diff --git a/hack/rpm/README.md b/hack/rpm/README.md index 1fb04bb8f..86a196de5 100644 --- a/hack/rpm/README.md +++ b/hack/rpm/README.md @@ -4,7 +4,7 @@ YUM and DNF (the replacement for YUM) use RPM packages for installation. This document describes how to build such packages for the Tanzu CLI, how to push them to a public repository and how to install the CLI from that repository. -There are two package names that can be built: +There are two package names that can be built by default: 1. "tanzu-cli" for official releases 2. "tanzu-cli-unstable" for pre-releases @@ -14,15 +14,22 @@ used; a version with a `-` in it is considered a pre-release and will use the `tanzu-cli-unstable` package name, while other versions will use the official `tanzu-cli` package name. +It is possible to specify the name of the package by setting the `RPM_PACKAGE_NAME` +environment variable to replace the default name of `tanzu-cli`. This should not +normally be used as the package name is what the end-users will install and the +default `tanzu-cli` name is the one users are familiar with. + ## Building the RPM package Executing the `hack/rpm/build_package.sh` script will build the RPM packages under `hack/rpm/_output`. The `hack/rpm/build_package.sh` script is meant to be run on a Linux machine that has `dnf` or `yum` installed. -This can be done in docker using the `fedora` image. To facilitate this -operation, the new `rpm-package` Makefile target has been added; this Makefile -target will first start a docker container and then run the -`hack/rpm/build_package.sh` script. +This can be done in docker using the `fedora` image. +Once the packages are built, the `hack/rpm/build_package_repo.sh` script should +be invoked to build the repository that will contain the packages. +To facilitate this double operation, the `rpm-package` Makefile target can be used; +this Makefile target will first start a docker container and then run the +appropriate scripts. The remote location of the existing repository can be overridden by setting the variable `RPM_METADATA_BASE_URI`. For example, the default value for diff --git a/hack/rpm/build_package.sh b/hack/rpm/build_package.sh index 71cd32da5..a29ecdd44 100755 --- a/hack/rpm/build_package.sh +++ b/hack/rpm/build_package.sh @@ -3,6 +3,11 @@ # Copyright 2022 VMware, Inc. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +# This script builds an RPM package for the Tanzu CLI for each supported architecture. +# It can be called more than once to build multiple packages with different names. +# This is useful if we want to use a different signing key for different packages. +# Once all the packages are built, the final repository can be built using the +# build_package_repo.sh script. set -e set -x @@ -26,6 +31,8 @@ fi # Strip 'v' prefix as an rpm package version must start with an integer VERSION=${VERSION#v} +# Set the default package name if not provided +RPM_PACKAGE_NAME=${RPM_PACKAGE_NAME:-"tanzu-cli"} BASE_DIR=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd) OUTPUT_DIR=${BASE_DIR}/_output/rpm/tanzu-cli @@ -35,7 +42,7 @@ ROOT_DIR=${BASE_DIR}/../.. # Install build dependencies if ! command -v rpmlint &> /dev/null; then - $DNF install -y rpmlint createrepo rpm-build yum-utils + $DNF install -y rpmlint rpm-build fi rpmlint ${BASE_DIR}/tanzu-cli.spec @@ -43,16 +50,23 @@ rpmlint ${BASE_DIR}/tanzu-cli.spec # We must create the sources directory ourselves in the below location mkdir -p ${HOME}/rpmbuild/SOURCES -# Create the .rpm packages -rm -rf ${OUTPUT_DIR} +# We support building multiple packages by calling the script multiple times +# and specifying a different RPM_PACKAGE_NAME. This is useful if we want +# to use a different signing key for different packages. +# So, we only want to clean the output directory if we have built +# the final repository, which indicates that this is an old build. +if [ -d ${OUTPUT_DIR}/repodata ]; then + rm -rf ${OUTPUT_DIR} +fi mkdir -p ${OUTPUT_DIR} mkdir -p ${PKG_DIR} cd ${ROOT_DIR} -UNSTABLE="false" # Transform the CLI version into RPM-compatible package version and release numbers. if [[ ${VERSION} == *-* ]]; then - UNSTABLE="true" + # If the version contains a - character, we are dealing with an unstable version + # so we should append -unstable to the package name + RPM_PACKAGE_NAME=${RPM_PACKAGE_NAME}-unstable # When there is a - in the version, we are dealing with a pre-release # e.g., 1.0.0-dev, 1.0.0-alpha.0, 1.0.0.rc.1, etc @@ -78,10 +92,10 @@ fi # Build the package for each architecture for arch in x86_64 aarch64; do - rpmbuild --define "rpm_package_version ${RPM_PACKAGE_VERSION}" \ + rpmbuild --define "rpm_package_name ${RPM_PACKAGE_NAME}" \ + --define "rpm_package_version ${RPM_PACKAGE_VERSION}" \ --define "rpm_release_version ${RPM_RELEASE_VERSION}" \ --define "cli_version v${VERSION}" \ - --define "unstable ${UNSTABLE}" \ -bb ${BASE_DIR}/tanzu-cli.spec \ --target ${arch} mv ${HOME}/rpmbuild/RPMS/${arch}/* ${PKG_DIR}/ @@ -92,52 +106,3 @@ for arch in x86_64 aarch64; do echo skip rpmsigning packages for ${arch} fi done - -###################### -# Build the repository -###################### - -# Prepare the existing repository info so we can sync from it -RPM_METADATA_BASE_URI=${RPM_METADATA_BASE_URI:=https://storage.googleapis.com/tanzu-cli-installer-packages} -RPM_REPO_GPG_PUBLIC_KEY_URI=${RPM_REPO_GPG_PUBLIC_KEY_URI:=https://storage.googleapis.com/tanzu-cli-installer-packages/keys/TANZU-PACKAGING-GPG-RSA-KEY.gpg} -if [ "${RPM_METADATA_BASE_URI}" = "new" ]; then - echo - echo "Building a brand new repository" - echo -else - cat << EOF | tee /tmp/tanzu-cli.repo -[tanzu-cli] -name=Tanzu CLI -baseurl=${RPM_METADATA_BASE_URI}/rpm/tanzu-cli -enabled=1 -gpgcheck=1 -repo_gpgcheck=1 -gpgkey=${RPM_REPO_GPG_PUBLIC_KEY_URI} -EOF - - # Sync the metadata so we can update it - # Use the --source flag to avoid downloading the actual RPMs - reposync --repoid=tanzu-cli --download-metadata -p ${OUTPUT_DIR} -c /tmp/tanzu-cli.repo --norepopath --source -y - # Remove the old signature, which won't be valid anymore - rm -f ${OUTPUT_DIR}/repodata/repomd.xml.asc - - # Now list the existing RPMs so we can pretend to have them locally - for p in $(reposync --repoid=tanzu-cli -c /tmp/tanzu-cli.repo -u -y | grep ${RPM_METADATA_BASE_URI}); do - echo "Found package: $p" - touch ${PKG_DIR}/$(basename $p) - done -fi - -# Create the repository metadata -createrepo --update --skip-stat ${OUTPUT_DIR} - -# Now that the repo is created, remove the fake empty packages so they don't -# risk being copied over the real ones in the final repository. -find ${PKG_DIR} -type f -empty -delete - -if [[ ! -z "${RPM_SIGNER}" ]]; then - # instead of ... gpg --detach-sign --armor repodata/repomd.xml - ${RPM_SIGNER} ${OUTPUT_DIR}/repodata/repomd.xml -else - echo skip rpmsigning repo -fi diff --git a/hack/rpm/build_package_repo.sh b/hack/rpm/build_package_repo.sh new file mode 100755 index 000000000..fb4ae7b66 --- /dev/null +++ b/hack/rpm/build_package_repo.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +# Copyright 2024 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# This script expects the RPM packages to already be present in the _output/rpm/tanzu-cli directory +# It will create a repository in the same directory and sign it with the provided key +# If the RPM_SIGNER environment variable is not set, the repository will not be signed + +set -e +set -x + +if [ $(uname) != "Linux" ]; then + echo "This script must be run on a Linux system" + exit 1 +fi + +# Use DNF and if it is not installed fallback to YUM +DNF=$(command -v dnf || command -v yum || true) +if [ -z "$DNF" ]; then + echo "This script requires the presence of either DNF or YUM package manager" + exit 1 +fi + +BASE_DIR=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd) +OUTPUT_DIR=${BASE_DIR}/_output/rpm/tanzu-cli +# Directory where the packages are stored +PKG_DIR=${OUTPUT_DIR} +ROOT_DIR=${BASE_DIR}/../.. + +# Install build dependencies +if ! command -v createrepo &> /dev/null; then + $DNF install -y createrepo yum-utils +fi + +cd ${ROOT_DIR} + +###################### +# Build the repository +###################### + +# Prepare the existing repository info so we can sync from it +RPM_METADATA_BASE_URI=${RPM_METADATA_BASE_URI:=https://storage.googleapis.com/tanzu-cli-installer-packages} +RPM_REPO_GPG_PUBLIC_KEY_URI=${RPM_REPO_GPG_PUBLIC_KEY_URI:=https://storage.googleapis.com/tanzu-cli-installer-packages/keys/TANZU-PACKAGING-GPG-RSA-KEY.gpg} +if [ "${RPM_METADATA_BASE_URI}" = "new" ]; then + echo + echo "Building a brand new repository" + echo +else + cat << EOF | tee /tmp/tanzu-cli.repo +[tanzu-cli] +name=Tanzu CLI +baseurl=${RPM_METADATA_BASE_URI}/rpm/tanzu-cli +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=${RPM_REPO_GPG_PUBLIC_KEY_URI} +EOF + + # Sync the metadata so we can update it + # Use the --source flag to avoid downloading the actual RPMs + reposync --repoid=tanzu-cli --download-metadata -p ${OUTPUT_DIR} -c /tmp/tanzu-cli.repo --norepopath --source -y + # Remove the old signature, which won't be valid anymore + rm -f ${OUTPUT_DIR}/repodata/repomd.xml.asc + + # Now list the existing RPMs so we can pretend to have them locally + for p in $(reposync --repoid=tanzu-cli -c /tmp/tanzu-cli.repo -u -y | grep ${RPM_METADATA_BASE_URI}); do + echo "Found package: $p" + touch ${PKG_DIR}/$(basename $p) + done +fi + +# Create the repository metadata +createrepo --update --skip-stat ${OUTPUT_DIR} + +# Now that the repo is created, remove the fake empty packages so they don't +# risk being copied over the real ones in the final repository. +find ${PKG_DIR} -type f -empty -delete + +if [[ ! -z "${RPM_SIGNER}" ]]; then + # instead of ... gpg --detach-sign --armor repodata/repomd.xml + ${RPM_SIGNER} ${OUTPUT_DIR}/repodata/repomd.xml +else + echo skip rpmsigning repo +fi diff --git a/hack/rpm/tanzu-cli.spec b/hack/rpm/tanzu-cli.spec index ba282cade..9f2ac41c7 100644 --- a/hack/rpm/tanzu-cli.spec +++ b/hack/rpm/tanzu-cli.spec @@ -1,13 +1,4 @@ -%if "%{unstable}" == "false" -Name: tanzu-cli -Provides: tanzu-cli -Obsoletes: tanzu-cli < %{rpm_package_version} -%else -Name: tanzu-cli-unstable -Provides: tanzu-cli-unstable -Obsoletes: tanzu-cli-unstable < %{rpm_package_version} -%endif - +Name: %{rpm_package_name} Version: %{rpm_package_version} Release: %{rpm_release_version} License: Apache 2.0