Nhudson/image ci fix #3
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and push images | |
| on: | |
| pull_request: {} | |
| push: | |
| branches: ['main'] | |
| jobs: | |
| find-directories: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| short_sha: ${{ steps.versions.outputs.SHORT_SHA }} | |
| branch_name: ${{ steps.versions.outputs.BRANCH_NAME }} | |
| build_matrix: ${{ steps.append_pg_configs.outputs.build_matrix }} | |
| tags: ${{ steps.tags.outputs.tags }} | |
| steps: | |
| - name: Check out the repo | |
| uses: actions/checkout@v4 | |
| - name: Set version strings | |
| id: versions | |
| run: | | |
| echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| echo "BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT | |
| - name: Find directories with Dockerfiles that changed | |
| id: find_directories | |
| uses: ./.github/actions/find-changed-directories | |
| with: | |
| contains_the_file: Dockerfile | |
| # If the branch does not exist, then it will not | |
| # filter any directories containing the file. | |
| # This allows for filtering out unchanged directories | |
| # in a pull request, and using all directories on the release | |
| # or main branches. | |
| changed_relative_to_ref: origin/${{ github.base_ref || 'not-a-branch' }} | |
| - name: Append PostgreSQL configurations to matrix | |
| id: append_pg_configs | |
| run: | | |
| # Read the initial matrix from the output of a previous step | |
| INITIAL_MATRIX='${{ steps.find_directories.outputs.build_matrix }}' | |
| echo "Initial Matrix: $INITIAL_MATRIX" | |
| # Define PostgreSQL configurations to append | |
| PG_CONFIGS='[{"pg_release": "14.10", "pg_version": "14"}, {"pg_release": "15.3", "pg_version": "15"}, {"pg_release": "16.1", "pg_version": "16"}]' | |
| # Convert PG_CONFIGS to a format that jq can iterate over properly | |
| PG_CONFIGS_JQ=$(echo $PG_CONFIGS | jq '{pg: .}') | |
| # Iterate over each directory and create a new entry for each PostgreSQL version | |
| # MODIFIED_MATRIX=$(echo $INITIAL_MATRIX | jq --argjson pgConfigs "$PG_CONFIGS_JQ" '.include | map(. + {pg: $pgConfigs.pg[]}) | {include: .}') | |
| # echo "Modified Matrix: $MODIFIED_MATRIX" | |
| # # Use the new syntax for setting outputs | |
| # echo "build_matrix=$MODIFIED_MATRIX" >> $GITHUB_OUTPUT | |
| # Compact the JSON into a single line | |
| MODIFIED_MATRIX=$(echo $INITIAL_MATRIX | jq --argjson pgConfigs "$PG_CONFIGS_JQ" '.include | map(. + {pg: $pgConfigs.pg[]}) | {include: .}' | jq -c .) | |
| echo "Modified Matrix: $MODIFIED_MATRIX" | |
| # Correctly format the output for GitHub Actions | |
| echo "build_matrix=$MODIFIED_MATRIX" >> $GITHUB_OUTPUT | |
| - name: Determine which tags to publish | |
| id: tags_list | |
| run: | | |
| BRANCH_NAME="${{ steps.versions.outputs.BRANCH_NAME }}" | |
| if [ "${BRANCH_NAME}" == "main" ]; then | |
| echo "tag_latest=true" >> $GITHUB_OUTPUT | |
| echo "tag_cargo=true" >> $GITHUB_OUTPUT | |
| elif [[ "${BRANCH_NAME}" == release/* ]]; then | |
| echo "tag_cargo=true" >> $GITHUB_OUTPUT | |
| echo "tag_latest=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "tag_latest=false" >> $GITHUB_OUTPUT | |
| echo "tag_cargo=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Install TOML parser | |
| run: | | |
| set -xe | |
| wget https://github.com/freshautomations/stoml/releases/download/v0.7.1/stoml_linux_amd64 | |
| mv stoml_linux_amd64 stoml | |
| chmod +x stoml | |
| sudo mv stoml /usr/local/bin/ | |
| - name: Create whitespace-separated tags list | |
| id: tags | |
| run: | | |
| SHORT_SHA="${{ steps.versions.outputs.SHORT_SHA }}" | |
| TAGS='' | |
| if [ "${{ steps.tags_list.outputs.tag_cargo }}" == "true" ]; then | |
| echo "Cargo file detected, adding to tags" | |
| VERSION=$(stoml Cargo.toml package.version)-${SHORT_SHA} | |
| TAGS="$TAGS $VERSION" | |
| fi | |
| if [ "${{ steps.tags_list.outputs.tag_latest }}" == "true" ]; then | |
| TAGS="$TAGS latest" | |
| fi | |
| TAGS="$TAGS ${SHORT_SHA}" | |
| echo "tags=$TAGS" >> $GITHUB_OUTPUT | |
| - name: Debug outputs | |
| run: | | |
| echo "Short SHA: ${{ steps.versions.outputs.SHORT_SHA }}" | |
| echo "Branch Name: ${{ steps.versions.outputs.BRANCH_NAME }}" | |
| echo "Build Matrix: ${{ steps.append_pg_configs.outputs.build_matrix }}" | |
| echo "Tags: ${{ steps.tags.outputs.tags }}" | |
| determine-build-conditions: | |
| needs: find-directories | |
| runs-on: ubuntu-latest | |
| outputs: | |
| build_tembo_pg_slim: ${{ steps.check_changes.outputs.build_tembo_pg_slim }} | |
| build_standard_cnpg: ${{ steps.check_changes.outputs.build_standard_cnpg }} | |
| build_ml_cnpg: ${{ steps.check_changes.outputs.build_standard_cnpg }} | |
| build_dw_cnpg: ${{ steps.check_changes.outputs.build_standard_cnpg }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Check changes for building images | |
| id: check_changes | |
| run: | | |
| MATRIX_JSON='${{ needs.find-directories.outputs.build_matrix }}' | |
| TEMBO_PG_SLIM_CHANGED=$(echo $MATRIX_JSON | jq '[.include[] | select(.name == "tembo-pg-slim") | .path] | any') | |
| STANDARD_CNPG_CHANGED=$(echo $MATRIX_JSON | jq '[.include[] | select(.name == "standard-cnpg") | .path] | any') | |
| ML_CNPG_CHANGED=$(echo $MATRIX_JSON | jq '[.include[] | select(.name == "ml-cnpg") | .path] | any') | |
| DW_CNPG_CHANGED=$(echo $MATRIX_JSON | jq '[.include[] | select(.name == "dw-cnpg") | .path] | any') | |
| # Initialize flags to false | |
| BUILD_TEMBO_PG_SLIM=false | |
| BUILD_STANDARD_CNPG=false | |
| BUILD_ML_CNPG=false | |
| BUILD_DW_CNPG=false | |
| # Check conditions and set flags | |
| if [[ "$TEMBO_PG_SLIM_CHANGED" == "true" ]]; then | |
| BUILD_TEMBO_PG_SLIM=true | |
| BUILD_STANDARD_CNPG=true # tembo-pg-slim change affects standard-cnpg | |
| fi | |
| if [[ "$STANDARD_CNPG_CHANGED" == "true" ]]; then | |
| BUILD_STANDARD_CNPG=true | |
| BUILD_ML_CNPG=true # standard-cnpg change affects ml-cnpg | |
| BUILD_DW_CNPG=true # standard-cnpg change affects dw-cnpg | |
| fi | |
| if [[ "$ML_CNPG_CHANGED" == "true" ]]; then | |
| BUILD_ML_CNPG=true | |
| fi | |
| if [[ "$DW_CNPG_CHANGED" == "true" ]]; then | |
| BUILD_DW_CNPG=true | |
| fi | |
| # Output the final flags | |
| echo "build_tembo_pg_slim=$BUILD_TEMBO_PG_SLIM" >> $GITHUB_OUTPUT | |
| echo "build_standard_cnpg=$BUILD_STANDARD_CNPG" >> $GITHUB_OUTPUT | |
| echo "build_ml_cnpg=$BUILD_ML_CNPG" >> $GITHUB_OUTPUT | |
| echo "build_dw_cnpg=$BUILD_DW_CNPG" >> $GITHUB_OUTPUT | |
| - name: Debug outputs | |
| run: | | |
| echo "Build Slim: ${{ steps.check_changes.outputs.build_tembo_pg_slim }}" | |
| echo "Build Standard: ${{ steps.check_changes.outputs.build_standard_cnpg }}" | |
| echo "Build ML: ${{ steps.check_changes.outputs.build_ml_cnpg }}" | |
| echo "Build DW: ${{ steps.check_changes.outputs.build_dw_cnpg }}" | |
| build-tembo-pg-slim: | |
| needs: [find-directories, determine-build-conditions] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| runs-on: | |
| - self-hosted | |
| - dind | |
| - large-8x8 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{fromJson(needs.find-directories.outputs.build_matrix)}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Build and push tembo-pg-slim image | |
| if: ${{ needs.determine-build-conditions.outputs.build_tembo_pg_slim == 'true' && matrix.name == 'tembo-pg-slim' }} | |
| run: | | |
| IMAGE_NAME=${{ matrix.name }}:pg${{ matrix.pg.pg_version }} | |
| docker build ${{ matrix.path }} --build-arg PG_RELEASE=${{ matrix.pg.pg_release }} --build-arg PG_VERSION=${{ matrix.pg.pg_version }} -t $IMAGE_NAME | |
| # Tag with each tag in the comma-separate list | |
| IFS=' ' read -ra TAG_ARRAY <<< "${{ needs.find-directories.outputs.tags }}" | |
| for tag in "${TAG_ARRAY[@]}"; do | |
| docker tag $IMAGE_NAME $IMAGE_NAME-$tag | |
| done | |
| - name: Build Docker images based on conditions | |
| run: | | |
| MATRIX_JSON='${{ needs.find-directories.outputs.build_matrix }}' | |
| BUILD_TEMBO_PG_SLIM=${{ needs.determine-build-conditions.outputs.build_tembo_pg_slim }} | |
| BUILD_STANDARD_CNPG=${{ needs.determine-build-conditions.outputs.build_standard_cnpg }} | |
| BUILD_ML_CNPG=${{ needs.determine-build-conditions.outputs.build_ml_cnpg }} | |
| BUILD_DW_CNPG=${{ needs.determine-build-conditions.outputs.build_dw_cnpg }} | |
| if [ "$BUILD_TEMBO_PG_SLIM" == "true" ]; then | |
| IMAGE_NAME=${{ matrix.name }}:pg${{ matrix.pg.pg_version }} | |
| docker build ${{ matrix.path }} --build-arg PG_RELEASE=${{ matrix.pg.pg_release }} --build-arg PG_VERSION=${{ matrix.pg.pg_version }} -t $IMAGE_NAME | |
| IFS=' ' read -ra TAG_ARRAY <<< "${{ needs.find-directories.outputs.tags }}" | |
| for tag in "${TAG_ARRAY[@]}"; do | |
| docker tag $IMAGE_NAME $IMAGE_NAME-$tag | |
| done | |
| docker images | |
| fi | |
| shell: bash | |
| # build-images: | |
| # needs: find-directories | |
| # permissions: | |
| # id-token: write | |
| # contents: read | |
| # runs-on: | |
| # - self-hosted | |
| # - dind | |
| # - large-8x8 | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: ${{fromJson(needs.find-directories.outputs.build_matrix)}} | |
| # - name: Build and Push Docker Image | |
| # run: | | |
| # IMAGE_NAME="${{ matrix.name }}:${{ matrix.pg.pg_version }}-${{ github.run_id }}" | |
| # docker build ${{ matrix.path }} --build-arg PG_RELEASE=${{ matrix.pg.pg_release }} --build-arg PG_VERSION=${{ matrix.pg.pg_version }} -t $IMAGE_NAME | |
| # docker push $IMAGE_NAME | |
| # env: | |
| # DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} | |
| # DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} | |
| # build_and_push: | |
| # name: Build and push images | |
| # permissions: | |
| # id-token: write | |
| # contents: read | |
| # runs-on: | |
| # - self-hosted | |
| # - dind | |
| # - large-8x8 | |
| # needs: | |
| # - find-directories | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: ${{ fromJson(needs.find-directories.outputs.build_images) }} | |
| # outputs: | |
| # short_sha: ${{ steps.versions.outputs.SHORT_SHA }} | |
| # steps: | |
| # - name: Check out the repo | |
| # uses: actions/checkout@v3 | |
| # - name: Set version strings | |
| # id: versions | |
| # run: | | |
| # echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| # - name: Determine which tags to publish | |
| # id: tags | |
| # run: | | |
| # BRANCH_NAME="${{ needs.find-directories.outputs.branch_name }}" | |
| # if [ "${BRANCH_NAME}" == "main" ]; then | |
| # echo "tag_latest=true" >> $GITHUB_OUTPUT | |
| # echo "tag_cargo=true" >> $GITHUB_OUTPUT | |
| # elif [[ "${BRANCH_NAME}" == release/* ]]; then | |
| # echo "tag_cargo=true" >> $GITHUB_OUTPUT | |
| # echo "tag_latest=false" >> $GITHUB_OUTPUT | |
| # else | |
| # echo "tag_latest=false" >> $GITHUB_OUTPUT | |
| # echo "tag_cargo=false" >> $GITHUB_OUTPUT | |
| # fi | |
| # - name: Build and upload image | |
| # uses: ./.github/actions/build-and-push-to-quay | |
| # with: | |
| # image_name: ${{ matrix.name }} | |
| # docker_directory: ${{ matrix.path }} | |
| # registry: "quay.io/tembo" | |
| # tag_cargo_version_if_present: ${{ steps.tags.outputs.tag_cargo }} | |
| # publish_latest: ${{ steps.tags.outputs.tag_latest }} | |
| # publish_calver: ${{ steps.tags_outputs.tag_latest }} | |
| # quay_user: ${{ secrets.QUAY_USER_TEMBO }} | |
| # quay_password: ${{ secrets.QUAY_PASSWORD_TEMBO }} | |
| # quay_user_tembo: ${{ secrets.QUAY_USER_TEMBO }} | |
| # quay_password_tembo: ${{ secrets.QUAY_PASSWORD_TEMBO }} | |
| # gha_iam_role: ${{ secrets.GHA_IAM_ROLE }} | |
| # ecr_registry: ${{ secrets.ECR_REGISTRY }} | |
| # pg_version: ${{ matrix.pg_version }} |