Fix stella embeddings + Integration tests for lorax (#668) #828
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and push docker image to ghcr and run integration tests | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - 'main' | |
| tags: | |
| - 'v*' | |
| jobs: | |
| build-test-push-image: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| runs-on: a100-40gb-runner | |
| permissions: | |
| contents: write | |
| packages: write | |
| # This is used to complete the identity challenge | |
| # with sigstore/fulcio when running outside of PRs. | |
| id-token: write | |
| security-events: write | |
| steps: | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v1 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GHCR_PAT }} | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: false | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: false | |
| swap-storage: true | |
| - name: Install soci | |
| uses: lerentis/[email protected] | |
| with: | |
| soci-release: 'v0.4.0' | |
| - name: Set up Docker Buildx | |
| uses: docker/[email protected] | |
| - name: Set up containerd for ubuntu | |
| uses: crazy-max/[email protected] | |
| with: | |
| config-inline: | | |
| version = 2 | |
| # persistent data location | |
| root = "/runner/build/containerd" | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| ghcr.io/predibase/lorax | |
| tags: | | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=sha,prefix=,suffix=,format=short | |
| type=raw,value=main,enable=${{ github.ref == 'refs/heads/main' }} | |
| - name: Create a hash from tags | |
| env: | |
| tags: ${{ steps.meta.outputs.tags }} | |
| id: vars | |
| run: | | |
| tag_hash=$(echo -n "$tags" | md5sum | awk '{print $1}') | |
| echo "tag_hash=$tag_hash" >> $GITHUB_OUTPUT | |
| echo "cache_dir=/runner/build/images/cache" >> $GITHUB_OUTPUT | |
| echo "image_dir=/runner/build/images" >> $GITHUB_OUTPUT | |
| echo "image_path=/runner/build/images/lorax" >> $GITHUB_OUTPUT | |
| - name: Create and update image/cache directory | |
| env: | |
| image_dir: ${{ steps.vars.outputs.image_dir }} | |
| cache_dir: ${{ steps.vars.outputs.cache_dir }} | |
| run: | | |
| sudo mkdir -p $image_dir | |
| sudo chown ubuntu:ubuntu $image_dir | |
| sudo mkdir -p $cache_dir | |
| sudo chown ubuntu:ubuntu $cache_dir | |
| - name: Export Docker image as OCI | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ./Dockerfile # Path to your Dockerfile | |
| push: false | |
| tags: ${{ steps.meta.outputs.tags }} | |
| load: true | |
| outputs: type=oci,compression=gzip,dest=${{ steps.vars.outputs.image_path }}-${{ steps.vars.outputs.tag_hash }}.tar.gz | |
| cache-from: type=local,src=${{ steps.vars.outputs.cache_dir }} | |
| cache-to: type=local,mode=max,image-manifest=true,oci-mediatypes=true,dest=${{ steps.vars.outputs.cache_dir }} | |
| # Uncomment below and comment the "Export Docker image as OCI" step | |
| # to test this workflow with an old image. This is useful for testing | |
| # the integration tests workflow without waiting for a new image to be built. | |
| # - name: Pull and retag an old image for testing with meta output tags | |
| # env: | |
| # tags: ${{ steps.meta.outputs.tags }} | |
| # run: | | |
| # # Read the first tag from meta output | |
| # docker pull ghcr.io/predibase/lorax:main | |
| # # Tag with all the meta output tags | |
| # while IFS= read -r tag; do | |
| # echo "Tagging $tag" | |
| # docker tag ghcr.io/predibase/lorax:main "$tag" | |
| # done <<< "$tags" | |
| - name: Run integration tests | |
| uses: ./.github/workflows/integration-tests | |
| with: | |
| test_image_tag: ${{ steps.meta.outputs.tags }} | |
| use_local_image: true | |
| github_token: ${{ secrets.GHCR_PAT }} | |
| huggingface_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} | |
| - name: Import image in containerd | |
| env: | |
| tag_hash: ${{ steps.vars.outputs.tag_hash }} | |
| image_path: ${{ steps.vars.outputs.image_path }} | |
| run: | | |
| echo "Importing $image_path-$tag_hash to Containerd" | |
| sudo ctr i import --no-unpack --all-platforms --digests $image_path-$tag_hash.tar.gz | |
| - name: Push image with containerd | |
| env: | |
| tags: ${{ steps.meta.outputs.tags }} | |
| run: | | |
| for tag in $tags | |
| do | |
| echo "Pushing $tag to GHCR" | |
| sudo ctr i push --user "${{ github.repository_owner }}:${{ secrets.GHCR_PAT }}" $tag | |
| done | |
| - name: Create and push soci index | |
| env: | |
| tags: ${{ steps.meta.outputs.tags }} | |
| run: | | |
| export SOCI_PATH=$HOME/.soci/soci | |
| for tag in $tags | |
| do | |
| echo "Creating soci index for $tag" | |
| sudo $SOCI_PATH create $tag | |
| echo "Pushing soci index for $tag" | |
| sudo $SOCI_PATH push --user ${{ github.repository_owner }}:${{ secrets.GHCR_PAT }} $tag | |
| done | |
| - name: Prune older images | |
| env: | |
| tag_hash: ${{ steps.vars.outputs.tag_hash }} | |
| image_path: ${{ steps.vars.outputs.image_path }} | |
| run: | | |
| # Delete images older than a day from docker store | |
| docker image prune -a -f --filter "until=24h" | |
| # Delete the on disk copy | |
| rm -rf "$image_path-$tag_hash.tar.gz" | |
| # Delete the SHA image(s) from containerd store | |
| sudo ctr i rm $(sudo ctr i ls -q) |