Skip to content

Use predibase API token when downloading pbase files (#688) #854

Use predibase API token when downloading pbase files (#688)

Use predibase API token when downloading pbase files (#688) #854

Workflow file for this run

name: Build and push docker image to ghcr and run integration tests
on:
workflow_dispatch:
push:
branches:
- 'main'
tags:
- "v*"
jobs:
build-test-push-image:
concurrency:
group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
runs-on: a100-40gb-runner
permissions:
contents: write
packages: write
# This is used to complete the identity challenge
# with sigstore/fulcio when running outside of PRs.
id-token: write
security-events: write
steps:
- name: Log in to GitHub Container Registry
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GHCR_PAT }}
- name: Checkout repository
uses: actions/checkout@v3
with:
submodules: recursive
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: false
swap-storage: true
- name: Install soci
uses: lerentis/[email protected]
with:
soci-release: "v0.4.0"
- name: Set up Docker Buildx
uses: docker/[email protected]
- name: Set up containerd for ubuntu
uses: crazy-max/[email protected]
with:
config-inline: |
version = 2
# persistent data location
root = "/runner/build/containerd"
- name: Generate SHA tag
id: meta
uses: docker/metadata-action@v5
with:
images: |
ghcr.io/predibase/lorax
tags: |
type=sha,prefix=,suffix=,format=short
- name: Create a hash from tags
env:
tags: ${{ steps.meta.outputs.tags }}
id: vars
run: |
tag_hash=$(echo -n "$tags" | md5sum | awk '{print $1}')
echo "tag_hash=$tag_hash" >> $GITHUB_OUTPUT
echo "cache_dir=/runner/build/images/cache" >> $GITHUB_OUTPUT
echo "image_dir=/runner/build/images" >> $GITHUB_OUTPUT
echo "image_path=/runner/build/images/lorax" >> $GITHUB_OUTPUT
- name: Create and update image/cache directory
env:
image_dir: ${{ steps.vars.outputs.image_dir }}
cache_dir: ${{ steps.vars.outputs.cache_dir }}
run: |
sudo mkdir -p $image_dir
sudo chown ubuntu:ubuntu $image_dir
sudo mkdir -p $cache_dir
sudo chown ubuntu:ubuntu $cache_dir
- name: Export Docker image as OCI
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile # Path to your Dockerfile
push: false
tags: ${{ steps.meta.outputs.tags }}
load: true
outputs: type=oci,compression=gzip,dest=${{ steps.vars.outputs.image_path }}-${{ steps.vars.outputs.tag_hash }}.tar.gz
cache-from: type=local,src=${{ steps.vars.outputs.cache_dir }}
cache-to: type=local,mode=max,image-manifest=true,oci-mediatypes=true,dest=${{ steps.vars.outputs.cache_dir }}
# Uncomment below and comment the "Export Docker image as OCI" step
# to test this workflow with an old image. This is useful for testing
# the integration tests workflow without waiting for a new image to be built.
# - name: Pull and retag an old image for testing with meta output tags
# env:
# tags: ${{ steps.meta.outputs.tags }}
# run: |
# # Read the first tag from meta output
# docker pull ghcr.io/predibase/lorax:main
# # Tag with all the meta output tags
# while IFS= read -r tag; do
# echo "Tagging $tag"
# docker tag ghcr.io/predibase/lorax:main "$tag"
# done <<< "$tags"
- name: Import image in containerd
env:
tag_hash: ${{ steps.vars.outputs.tag_hash }}
image_path: ${{ steps.vars.outputs.image_path }}
run: |
echo "Importing $image_path-$tag_hash to Containerd"
sudo ctr i import --no-unpack --all-platforms --digests $image_path-$tag_hash.tar.gz
- name: Push image with containerd
env:
tags: ${{ steps.meta.outputs.tags }}
run: |
for tag in $tags
do
echo "Pushing $tag to GHCR"
sudo ctr i push --user "${{ github.repository_owner }}:${{ secrets.GHCR_PAT }}" $tag
done
- name: Create and push soci index
env:
tags: ${{ steps.meta.outputs.tags }}
run: |
export SOCI_PATH=$HOME/.soci/soci
for tag in $tags
do
echo "Creating soci index for $tag"
sudo $SOCI_PATH create $tag
echo "Pushing soci index for $tag"
sudo $SOCI_PATH push --user ${{ github.repository_owner }}:${{ secrets.GHCR_PAT }} $tag
done
- name: Run integration tests
id: integration_tests
uses: ./.github/workflows/integration-tests
with:
test_image_tag: ${{ steps.meta.outputs.tags }}
use_local_image: true
github_token: ${{ secrets.GHCR_PAT }}
huggingface_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
- name: Final Tags
id: final_meta
uses: docker/metadata-action@v5
with:
images: |
ghcr.io/predibase/lorax
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix=,suffix=,format=short
type=raw,value=main,enable=${{ github.ref == 'refs/heads/main' }}
- name: Tag the image with rest of the tags and push
env:
sha_tag: ${{ steps.meta.outputs.tags }}
tags: ${{ steps.final_meta.outputs.tags }}
run: |
# Read the first tag from meta output
docker pull $sha_tag
# Tag with all the meta output tags
while IFS= read -r tag; do
echo "Tagging $tag and pushing"
docker tag $sha_tag "$tag"
docker push "$tag"
echo "Pushed $tag"
done <<< "$tags"
- name: Prune older images
env:
tag_hash: ${{ steps.vars.outputs.tag_hash }}
image_path: ${{ steps.vars.outputs.image_path }}
run: |
# Delete images older than a day from docker store
docker image prune -a -f --filter "until=24h"
# Delete the on disk copy
rm -rf "$image_path-$tag_hash.tar.gz"
# Delete the SHA image(s) from containerd store
sudo ctr i rm $(sudo ctr i ls -q)