Skip to content

Commit

Permalink
build build-env images similar to others
Browse files Browse the repository at this point in the history
  • Loading branch information
@daler
daler committed Jan 27, 2024
1 parent 04e6b30 commit 7388860
Show file tree
Hide file tree
Showing 3 changed files with 177 additions and 85 deletions.
251 changes: 171 additions & 80 deletions .github/workflows/build-env-image.yml
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
name: Build image
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

name: 'Build & Push: base-glibc-busybox-bash'
on:
pull_request:
paths:
Expand All @@ -14,20 +10,18 @@ on:

jobs:
build:
name: Build image - ${{ matrix.image }}
name: Build & Push
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- arch: arm64
image: bioconda/bioconda-utils-build-env-cos7-aarch64
base_image: quay.io/condaforge/linux-anvil-aarch64
- arch: amd64
image: bioconda/bioconda-utils-build-env-cos7-x86_64
base_image: quay.io/condaforge/linux-anvil-cos7-x86_64
env:
# The base image is not intended to change often and should be used with
# version tags or checksum IDs, but not via "latest".
MAJOR_VERSION: 3
MINOR_VERSION: 0
IMAGE_NAME: bioconda-utils-build-env-cos7

steps:
- name: Checkout bioconda-containers
uses: actions/checkout@v4
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Checkout bioconda-utils
Expand All @@ -37,81 +31,178 @@ jobs:
repository: 'bioconda/bioconda-utils'
path: 'bioconda-utils'

- id: get-tag
run: |
tag=${{ github.event.release && github.event.release.tag_name || github.sha }}
printf %s "tag=${tag#v}" >> $GITHUB_OUTPUT
- name: Install qemu dependency
if: ${{ matrix.arch == 'arm64' }}
uses: docker/setup-qemu-action@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
with:
platforms: arm64

- name: Build image
id: buildah-build
uses: redhat-actions/buildah-build@v2
with:
image: ${{ matrix.image }}
arch: ${{ matrix.arch }}
build-args: |
BASE_IMAGE=${{ matrix.base_image }}
tags: >-
- name: Build
id: build
run: |
set -xeu
cd 'images/${{ env.IMAGE_NAME }}'
image_name='${{ env.IMAGE_NAME }}'
tags='
${{ env.MAJOR_VERSION }}
${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }}
latest
${{ steps.get-tag.outputs.tag }}
dockerfiles: |
./images/build-env/Dockerfile
'
# Adds image and tags to outputs which can be used in later steps.
printf %s\\n \
"image=${image_name}" \
"tags=$( echo ${tags} )" \
>> $GITHUB_OUTPUT
# Create manifest (which is considered arch-independent)
for tag in ${tags} ; do
buildah manifest create "${image_name}:${tag}"
done
# Due to different nomenclature used by conda-forge and buildah, we
# need to map archs to base images.
for arch_and_image in \
"amd64=quay.io/condaforge/linux-anvil-cos7-x86_64" \
"arm64=quay.io/condaforge/linux-anvil-aarch64";
do
# Unpack archs and base images
arch=$(cut -f1 -d "=" $arch_and_image)
base_image=$(cut -f2 -d "=" $arch_and_image)
# --iidfile prints the built image ID to the specified file. This is
# used so we can refer to the image in later steps.
iidfile="$( mktemp )"
buildah bud \
--arch="${arch}" \
--iidfile="${iidfile}" \
--build-arg=base_image="$base_image" \
image_id="$( cat "${iidfile}" )"
rm "${iidfile}"
# Extract various package info and version info to store as labels
container="$( buildah from "${image_id}" )"
run() { buildah run "${container}" "${@}" ; }
deb_list="$( run cat /.deb.lst | tr '\n' '|' | sed 's/|$//' )"
pkg_list="$( run cat /.pkg.lst | tr '\n' '|' | sed 's/|$//' )"
glibc="$( run sh -c 'exec "$( find -xdev -name libc.so.6 -print -quit )"' | sed '1!d' )"
bash="$( run bash --version | sed '1!d' )"
bioconda_utils="$( run bioconda-utils --version )"
buildah rm "${container}"
# Store package/version info as labels for the image
container="$( buildah from "${image_id}" )"
buildah config \
--label=glibc="${glibc}" \
--label=bash="${bash}" \
--label=deb-list="${deb_list}" \
--label=pkg-list="${pkg_list}" \
--label=bioconda-utils="${bioconda_utils}" \
"${container}"
- name: Test built image
# Store the new image (now with labels)
image_id="$( buildah commit "${container}" )"
buildah rm "${container}"
# image tag includes arch; then added to manifest which does not include arch
for tag in ${tags} ; do
buildah tag \
"${image_id}" \
"${image_name}:${tag}-${arch}"
buildah manifest add \
"${image_name}:${tag}" \
"${image_id}"
done
done
- name: Test
run: |
image='${{ steps.buildah-build.outputs.image }}'
for tag in ${{ steps.buildah-build.outputs.tags }} ; do
podman run --rm "${image}:${tag}" bioconda-utils --version
image='${{ steps.build.outputs.image }}'
# Extract image ids from manifest to test.
ids="$(
for tag in ${{ steps.build.outputs.tags }} ; do
buildah manifest inspect "${image}:${tag}" \
| jq -r '.manifests[]|.digest' \
| while read id ; do
buildah images --format '{{.ID}}{{.Digest}}' \
| sed -n "s/${id}//p"
done
done
)"
# See Dockerfile.test for actual tests run
ids="$( printf %s "${ids}" | sort -u )"
for id in ${ids} ; do
podman history "${id}"
buildah bud \
--build-arg=base="${id}" \
--file=Dockerfile.test \
"images/${image}"
done
buildah rmi --prune || true
- name: Check Tags
run: |
# FIX upstream: Quay.io does not support immutable images currently.
# => Try to use the REST API to check for duplicate tags and exit if they exist
response="$(
curl -sL \
'https://quay.io/api/v1/repository/bioconda/${{ steps.build.outputs.image }}/tag/'
)"
- name: Push To Quay
if: github.ref == 'refs/heads/main' && github.repository == 'bioconda/bioconda-containers'
existing_tags="$(
printf %s "${response}" \
| jq -r '.tags[]|select(.end_ts == null or .end_ts >= now)|.name'
)" \
|| {
printf %s\\n \
'Could not get list of image tags.' \
'Does the repository exist on Quay.io?' \
'Quay.io REST API response was:' \
"${response}"
exit 1
}
for tag in ${{ steps.build.outputs.tags }} ; do
case "${tag}" in
latest | '${{ env.MAJOR_VERSION }}' ) ;;
* )
if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then
printf 'Tag %s already exists!\n' "${tag}"
exit 1
fi
esac
done
- if: ${{ github.ref == 'refs/heads/main' }}
name: Push
uses: redhat-actions/push-to-registry@v2
with:
image: ${{ steps.buildah-build.outputs.image }}
tags: ${{ steps.buildah-build.outputs.tags }}
image: ${{ steps.build.outputs.image }}
tags: ${{ steps.build.outputs.tags }}
registry: ${{ secrets.QUAY_BIOCONDA_REPO }}
username: ${{ secrets.QUAY_BIOCONDA_USERNAME }}
password: ${{ secrets.QUAY_BIOCONDA_TOKEN }}

build-manifest:
needs: [build]
if: github.ref == 'refs/heads/main' && github.repository == 'bioconda/bioconda-containers'
name: quay.io/bioconda/${{ matrix.cfg.DOCKER_MANIFEST }}:${{ matrix.cfg.DOCKER_TAG }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
cfg:
- DOCKER_MANIFEST: bioconda-utils-build-env-cos7
DOCKER_TAG: "latest"
DOCKER_IMAGES: "quay.io/<<USER>>/bioconda-utils-build-env-cos7:<<TAG>>,quay.io/<<USER>>/bioconda-utils-build-env-cos7-aarch64:<<TAG>>"

steps:
- uses: actions/checkout@v4

- name: Interpolate placeholders
id: interpolate
run: |
set -x
INTERPOLATED=`echo "${{ matrix.cfg.DOCKER_IMAGES }}" | sed "s#<<USER>>#${{ secrets.QUAY_BIOCONDA_USERNAME }}#g" | sed "s#<<TAG>>#${{ matrix.cfg.DOCKER_TAG }}#g"`
echo "DOCKER_IMAGES=${INTERPOLATED}" >> "$GITHUB_OUTPUT"
- name: Login to Quay.io registry
uses: docker/login-action@v2
with:
registry: ${{ secrets.QUAY_BIOCONDA_REPO }}
username: ${{ secrets.QUAY_BIOCONDA_USERNAME }}
password: ${{ secrets.QUAY_BIOCONDA_TOKEN }}

- name: Push Docker manifest list for quay.io/bioconda
uses: Noelware/[email protected]
with:
inputs: quay.io/${{ secrets.QUAY_BIOCONDA_USERNAME }}/${{ matrix.cfg.DOCKER_MANIFEST }}:${{ matrix.cfg.DOCKER_TAG }}
images: ${{ steps.interpolate.outputs.DOCKER_IMAGES }}
push: true
- if: ${{ github.ref == 'refs/heads/main' }}
name: Test Pushed
run: |
image='${{ env.IMAGE_NAME }}'
ids="$(
for tag in ${{ steps.build.outputs.tags }} ; do
buildah manifest inspect "${image}:${tag}" \
| jq -r '.manifests[]|.digest' \
| while read id ; do
buildah images --format '{{.ID}}{{.Digest}}' \
| sed -n "s/${id}//p"
done
done
)"
ids="$( printf %s "${ids}" | sort -u )"
for id in ${ids} ; do
podman history "${id}"
buildah bud \
--build-arg=base="${id}" \
--file=Dockerfile.test \
"images/${image}"
done
buildah rmi --prune || true
7 changes: 2 additions & 5 deletions images/build-env/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
# Specify the base image to support multi-arch images, such as
# - 'quay.io/condaforge/linux-anvil-aarch64' for Linux aarch64
# - 'quay.io/condaforge/linux-anvil-cos7-x86_64' for Linux x86_64
ARG BASE_IMAGE=quay.io/condaforge/linux-anvil-cos7-x86_64
ARG base_image

FROM ${BASE_IMAGE} as base
FROM ${base_image} as base

# Copy over C.UTF-8 locale from our base image to make it consistently available during build.
COPY --from=quay.io/bioconda/base-glibc-busybox-bash /usr/lib/locale/C.utf8 /usr/lib/locale/C.utf8
Expand Down
4 changes: 4 additions & 0 deletions images/build-env/Dockerfile.test
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
ARG base
FROM "${base}"
RUN bioconda-utils --version
RUN bioconda-utils --help

0 comments on commit 7388860

Please sign in to comment.