build_cloud_images.yaml

name: Build and push images

on:
  pull_request: {}
  push:
    branches: ['main']

jobs:

  find-directories:
    runs-on: ubuntu-latest
    outputs:
      short_sha: ${{ steps.versions.outputs.SHORT_SHA }}
      branch_name: ${{ steps.versions.outputs.BRANCH_NAME }}
      build_matrix: ${{ steps.append_pg_configs.outputs.build_matrix }}
      tags: ${{ steps.tags.outputs.tags }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v4
      - name: Set version strings
        id: versions
        run: |
          echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
          echo "BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT
      - name: Find directories with Dockerfiles that changed
        id: find_directories
        uses: ./.github/actions/find-changed-directories
        with:
          contains_the_file: Dockerfile
          # If the branch does not exist, then it will not
          # filter any directories containing the file.
          # This allows for filtering out unchanged directories
          # in a pull request, and using all directories on the release
          # or main branches.
          changed_relative_to_ref: origin/${{ github.base_ref || 'not-a-branch' }}
      - name: Append PostgreSQL configurations to matrix
        id: append_pg_configs
        run: |
          # Read the initial matrix from the output of a previous step
          INITIAL_MATRIX='${{ steps.find_directories.outputs.build_matrix }}'
          echo "Initial Matrix: $INITIAL_MATRIX"
          # Define PostgreSQL configurations to append
          PG_CONFIGS='[{"pg_release": "14.10", "pg_version": "14"}, {"pg_release": "15.3", "pg_version": "15"}, {"pg_release": "16.1", "pg_version": "16"}]'
          # Convert PG_CONFIGS to a format that jq can iterate over properly
          PG_CONFIGS_JQ=$(echo $PG_CONFIGS | jq '{pg: .}')
          # Iterate over each directory and create a new entry for each PostgreSQL version
          # MODIFIED_MATRIX=$(echo $INITIAL_MATRIX | jq --argjson pgConfigs "$PG_CONFIGS_JQ" '.include | map(. + {pg: $pgConfigs.pg[]}) | {include: .}')
          # echo "Modified Matrix: $MODIFIED_MATRIX"
          # # Use the new syntax for setting outputs
          # echo "build_matrix=$MODIFIED_MATRIX" >> $GITHUB_OUTPUT
          # Compact the JSON into a single line
          MODIFIED_MATRIX=$(echo $INITIAL_MATRIX | jq --argjson pgConfigs "$PG_CONFIGS_JQ" '.include | map(. + {pg: $pgConfigs.pg[]}) | {include: .}' | jq -c .)
          echo "Modified Matrix: $MODIFIED_MATRIX"
          # Correctly format the output for GitHub Actions
          echo "build_matrix=$MODIFIED_MATRIX" >> $GITHUB_OUTPUT
      - name: Determine which tags to publish
        id: tags_list
        run: |
          BRANCH_NAME="${{ steps.versions.outputs.BRANCH_NAME }}"
          if [ "${BRANCH_NAME}" == "main" ]; then
            echo "tag_latest=true" >> $GITHUB_OUTPUT
            echo "tag_cargo=true" >> $GITHUB_OUTPUT
          elif [[ "${BRANCH_NAME}" == release/* ]]; then
            echo "tag_cargo=true" >> $GITHUB_OUTPUT
            echo "tag_latest=false" >> $GITHUB_OUTPUT
          else
            echo "tag_latest=false" >> $GITHUB_OUTPUT
            echo "tag_cargo=false" >> $GITHUB_OUTPUT
          fi
      - name: Install TOML parser
        run: |
         set -xe
         wget https://github.com/freshautomations/stoml/releases/download/v0.7.1/stoml_linux_amd64
         mv stoml_linux_amd64 stoml
         chmod +x stoml
         sudo mv stoml /usr/local/bin/
      - name: Create whitespace-separated tags list
        id: tags
        run: |
          SHORT_SHA="${{ steps.versions.outputs.SHORT_SHA }}"
          TAGS=''
          if [ "${{ steps.tags_list.outputs.tag_cargo }}" == "true" ]; then
              echo "Cargo file detected, adding to tags"
              VERSION=$(stoml Cargo.toml package.version)-${SHORT_SHA}
              TAGS="$TAGS $VERSION"
          fi
          if [ "${{ steps.tags_list.outputs.tag_latest }}" == "true" ]; then
            TAGS="$TAGS latest"
          fi
          TAGS="$TAGS ${SHORT_SHA}"
          echo "tags=$TAGS" >> $GITHUB_OUTPUT
      - name: Debug outputs
        run: |
          echo "Short SHA: ${{ steps.versions.outputs.SHORT_SHA }}"
          echo "Branch Name: ${{ steps.versions.outputs.BRANCH_NAME }}"
          echo "Build Matrix: ${{ steps.append_pg_configs.outputs.build_matrix }}"
          echo "Tags: ${{ steps.tags.outputs.tags }}"

  determine-build-conditions:
    needs: find-directories
    runs-on: ubuntu-latest
    outputs:
      build_tembo_pg_slim: ${{ steps.check_changes.outputs.build_tembo_pg_slim }}
      build_standard_cnpg: ${{ steps.check_changes.outputs.build_standard_cnpg }}
      build_ml_cnpg: ${{ steps.check_changes.outputs.build_standard_cnpg }}
      build_dw_cnpg: ${{ steps.check_changes.outputs.build_standard_cnpg }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Check changes for building images
        id: check_changes
        run: |
          MATRIX_JSON='${{ needs.find-directories.outputs.build_matrix }}'
          TEMBO_PG_SLIM_CHANGED=$(echo $MATRIX_JSON | jq '[.include[] | select(.name == "tembo-pg-slim") | .path] | any')
          STANDARD_CNPG_CHANGED=$(echo $MATRIX_JSON | jq '[.include[] | select(.name == "standard-cnpg") | .path] | any')
          ML_CNPG_CHANGED=$(echo $MATRIX_JSON | jq '[.include[] | select(.name == "ml-cnpg") | .path] | any')
          DW_CNPG_CHANGED=$(echo $MATRIX_JSON | jq '[.include[] | select(.name == "dw-cnpg") | .path] | any')

          # Initialize flags to false
          BUILD_TEMBO_PG_SLIM=false
          BUILD_STANDARD_CNPG=false
          BUILD_ML_CNPG=false
          BUILD_DW_CNPG=false

          # Check conditions and set flags
          if [[ "$TEMBO_PG_SLIM_CHANGED" == "true" ]]; then
            BUILD_TEMBO_PG_SLIM=true
            BUILD_STANDARD_CNPG=true # tembo-pg-slim change affects standard-cnpg
          fi

          if [[ "$STANDARD_CNPG_CHANGED" == "true" ]]; then
            BUILD_STANDARD_CNPG=true
            BUILD_ML_CNPG=true # standard-cnpg change affects ml-cnpg
            BUILD_DW_CNPG=true # standard-cnpg change affects dw-cnpg
          fi

          if [[ "$ML_CNPG_CHANGED" == "true" ]]; then
            BUILD_ML_CNPG=true
          fi

          if [[ "$DW_CNPG_CHANGED" == "true" ]]; then
            BUILD_DW_CNPG=true
          fi

          # Output the final flags
          echo "build_tembo_pg_slim=$BUILD_TEMBO_PG_SLIM" >> $GITHUB_OUTPUT
          echo "build_standard_cnpg=$BUILD_STANDARD_CNPG" >> $GITHUB_OUTPUT
          echo "build_ml_cnpg=$BUILD_ML_CNPG" >> $GITHUB_OUTPUT
          echo "build_dw_cnpg=$BUILD_DW_CNPG" >> $GITHUB_OUTPUT
      - name: Debug outputs
        run: |
          echo "Build Slim: ${{ steps.check_changes.outputs.build_tembo_pg_slim }}"
          echo "Build Standard: ${{ steps.check_changes.outputs.build_standard_cnpg }}"
          echo "Build ML: ${{ steps.check_changes.outputs.build_ml_cnpg }}"
          echo "Build DW: ${{ steps.check_changes.outputs.build_dw_cnpg }}"

  build-tembo-pg-slim:
    needs: [find-directories, determine-build-conditions]
    permissions:
      id-token: write
      contents: read
    runs-on:
      - self-hosted
      - dind
      - large-8x8
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.find-directories.outputs.build_matrix)}}
    if: ${{ needs.determine-build-conditions.outputs.build_tembo_pg_slim == 'true' && contains(matrix.name, 'tembo-pg-slim') }}
    steps:
      - uses: actions/checkout@v4
      - name: Build and push tembo-pg-slim image
        run: |
          IMAGE_NAME=${{ matrix.name }}:pg${{ matrix.pg.pg_version }}
          docker build ${{ matrix.path }} --build-arg PG_RELEASE=${{ matrix.pg.pg_release }} --build-arg PG_VERSION=${{ matrix.pg.pg_version }} -t $IMAGE_NAME
          # Tag with each tag in the comma-separate list
          IFS=' ' read -ra TAG_ARRAY <<< "${{ needs.find-directories.outputs.tags }}"
          for tag in "${TAG_ARRAY[@]}"; do
            docker tag $IMAGE_NAME $IMAGE_NAME-$tag
          done

  # build-images:
  #   needs: find-directories
  #   permissions:
  #     id-token: write
  #     contents: read
  #   runs-on:
  #     - self-hosted
  #     - dind
  #     - large-8x8
  #   strategy:
  #     fail-fast: false
  #     matrix: ${{fromJson(needs.find-directories.outputs.build_matrix)}}
      # - name: Build and Push Docker Image
      #   run: |
      #     IMAGE_NAME="${{ matrix.name }}:${{ matrix.pg.pg_version }}-${{ github.run_id }}"
      #     docker build ${{ matrix.path }} --build-arg PG_RELEASE=${{ matrix.pg.pg_release }} --build-arg PG_VERSION=${{ matrix.pg.pg_version }} -t $IMAGE_NAME
      #     docker push $IMAGE_NAME
      #   env:
      #     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      #     DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
  # build_and_push:
  #   name: Build and push images
  #   permissions:
  #     id-token: write
  #     contents: read
  #   runs-on:
  #     - self-hosted
  #     - dind
  #     - large-8x8
  #   needs:
  #     - find-directories
  #   strategy:
  #     fail-fast: false
  #     matrix: ${{ fromJson(needs.find-directories.outputs.build_images) }}
  #   outputs:
  #       short_sha: ${{ steps.versions.outputs.SHORT_SHA }}
  #   steps:
  #     - name: Check out the repo
  #       uses: actions/checkout@v3
  #     - name: Set version strings
  #       id: versions
  #       run: |
  #           echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
  #     - name: Determine which tags to publish
  #       id: tags
  #       run: |
  #         BRANCH_NAME="${{ needs.find-directories.outputs.branch_name }}"
  #         if [ "${BRANCH_NAME}" == "main" ]; then
  #           echo "tag_latest=true" >> $GITHUB_OUTPUT
  #           echo "tag_cargo=true" >> $GITHUB_OUTPUT
  #         elif [[ "${BRANCH_NAME}" == release/* ]]; then
  #           echo "tag_cargo=true" >> $GITHUB_OUTPUT
  #           echo "tag_latest=false" >> $GITHUB_OUTPUT
  #         else
  #           echo "tag_latest=false" >> $GITHUB_OUTPUT
  #           echo "tag_cargo=false" >> $GITHUB_OUTPUT
  #         fi
  #     - name: Build and upload image
  #       uses: ./.github/actions/build-and-push-to-quay
  #       with:
  #           image_name: ${{ matrix.name }}
  #           docker_directory: ${{ matrix.path }}
  #           registry: "quay.io/tembo"
  #           tag_cargo_version_if_present: ${{ steps.tags.outputs.tag_cargo }}
  #           publish_latest: ${{ steps.tags.outputs.tag_latest }}
  #           publish_calver: ${{ steps.tags_outputs.tag_latest }}
  #           quay_user: ${{ secrets.QUAY_USER_TEMBO }}
  #           quay_password: ${{ secrets.QUAY_PASSWORD_TEMBO }}  
  #           quay_user_tembo: ${{ secrets.QUAY_USER_TEMBO }}
  #           quay_password_tembo: ${{ secrets.QUAY_PASSWORD_TEMBO }} 
  #           gha_iam_role: ${{ secrets.GHA_IAM_ROLE }}
  #           ecr_registry: ${{ secrets.ECR_REGISTRY }}
  #           pg_version: ${{ matrix.pg_version }}