test-build-deploy.yml

name: ci
permissions: read-all
on:
  push:
    branches: [master]
    tags:
      - v[0-9]+.[0-9]+.[0-9]+** # Tag filters not as strict due to different regex system on Github Actions
    paths-ignore:
      - 'build-image/**'
      - '.github/workflows/build-image.yml'
  pull_request:
    paths-ignore:
      - 'build-image/**'
      - '.github/workflows/build-image.yml'

jobs:
  lint:
    runs-on: ubuntu-20.04
    container:
      image: quay.io/cortexproject/build-image:master-0ddced051
    steps:
      - name: Checkout Repo
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
        # Commands in the Makefile are hardcoded with an assumed file structure of the CI container
        # Symlink ensures paths specified in previous commands don’t break
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Lint
        run: make BUILD_IN_CONTAINER=false lint
      - name: Check Vendor Directory
        run: make BUILD_IN_CONTAINER=false mod-check
      - name: Check Protos
        run: make BUILD_IN_CONTAINER=false check-protos
      - name: Check Generated Documentation
        run: make BUILD_IN_CONTAINER=false check-doc
      - name: Check White Noise.
        run: make BUILD_IN_CONTAINER=false check-white-noise

  test:
    runs-on: ubuntu-20.04
    container:
      image: quay.io/cortexproject/build-image:master-0ddced051
    steps:
      - name: Checkout Repo
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Run Tests
        run: make BUILD_IN_CONTAINER=false test
  test-no-race:
    runs-on: ubuntu-20.04
    container:
      image: quay.io/cortexproject/build-image:master-0ddced051
    steps:
      - name: Checkout Repo
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Run Tests
        run: make BUILD_IN_CONTAINER=false test-no-race

  security:
    name: CodeQL
    runs-on: ubuntu-20.04
    permissions:
      actions: read
      contents: read
      security-events: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
        uses: github/codeql-action/init@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9
        with:
          languages: go

      - name: Autobuild
        uses: github/codeql-action/autobuild@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9

  
  build:
    runs-on: ubuntu-20.04
    container:
      image: quay.io/cortexproject/build-image:master-0ddced051
    steps:
      - name: Checkout Repo
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Install Docker Client
        run: ./.github/workflows/scripts/install-docker.sh
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Build Image
        run: |
          touch build-image/.uptodate
          make BUILD_IN_CONTAINER=false
      - name: Build Website
        run: |
          touch build-image/.uptodate
          make BUILD_IN_CONTAINER=false web-build
      - name: Upload Website Artifact
        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
        with:
          name: website public
          path: website/public/
      - name: Save Images
        run: |
          mkdir /tmp/images
          ln -s /tmp/images ./docker-images
          make BUILD_IN_CONTAINER=false save-images
      - name: Create Docker Images Archive
        run: tar -cvf images.tar /tmp/images
      - name: Upload Docker Images Artifact
        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
        with:
          name: Docker Images
          path: ./images.tar

  integration:
    needs: build
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        tags:
          - requires_docker
          - integration_alertmanager
          - integration_backward_compatibility
          - integration_memberlist
          - integration_querier
          - integration_ruler
          - integration_query_fuzz
    steps:
      - name: Upgrade golang
        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
        with:
          go-version: 1.23.2
      - name: Checkout Repo
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Install Docker Client
        run: sudo ./.github/workflows/scripts/install-docker.sh
      - name: Sym Link Expected Path to Workspace
        run: |
          sudo mkdir -p /go/src/github.com/cortexproject/cortex
          sudo ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Download Docker Images Artifacts
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          name: Docker Images
      - name: Extract Docker Images Archive
        run: tar -xvf images.tar -C /
      - name: Load Docker Images
        run: |
          ln -s /tmp/images ./docker-images
          make BUILD_IN_CONTAINER=false load-images
      - name: Preload Images
        # We download docker images used by integration tests so that all images are available
        # locally and the download time doesn't account in the test execution time, which is subject
        # to a timeout
        run: |
          docker pull minio/minio:RELEASE.2024-05-28T17-19-04Z
          docker pull consul:1.8.4
          docker pull gcr.io/etcd-development/etcd:v3.4.7
          if [ "$TEST_TAGS" = "integration_backward_compatibility" ]; then
            docker pull quay.io/cortexproject/cortex:v1.13.1
            docker pull quay.io/cortexproject/cortex:v1.13.2
            docker pull quay.io/cortexproject/cortex:v1.14.0
            docker pull quay.io/cortexproject/cortex:v1.14.1
            docker pull quay.io/cortexproject/cortex:v1.15.0
            docker pull quay.io/cortexproject/cortex:v1.15.1
            docker pull quay.io/cortexproject/cortex:v1.15.2
            docker pull quay.io/cortexproject/cortex:v1.15.3
            docker pull quay.io/cortexproject/cortex:v1.16.0
            docker pull quay.io/cortexproject/cortex:v1.16.1
            docker pull quay.io/cortexproject/cortex:v1.17.0
            docker pull quay.io/cortexproject/cortex:v1.17.1
            docker pull quay.io/cortexproject/cortex:v1.18.0
            docker pull quay.io/cortexproject/cortex:v1.18.1
          elif [ "$TEST_TAGS" = "integration_query_fuzz" ]; then
            docker pull quay.io/cortexproject/cortex:v1.18.1 
            docker pull quay.io/prometheus/prometheus:v2.51.0
            docker pull quay.io/prometheus/prometheus:v2.55.1
          fi
          docker pull memcached:1.6.1
          docker pull redis:7.0.4-alpine
        env:
          TEST_TAGS: ${{ matrix.tags }}
      - name: Integration Tests
        run: |
          export CORTEX_IMAGE_PREFIX="${IMAGE_PREFIX:-quay.io/cortexproject/}"
          export IMAGE_TAG=$(make image-tag)
          export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:$IMAGE_TAG-amd64"
          export CORTEX_CHECKOUT_DIR="/go/src/github.com/cortexproject/cortex"
          echo "Running integration tests with image: $CORTEX_IMAGE"
          go test -tags=integration,${{ matrix.tags }} -timeout 2400s -v -count=1 ./integration/...
        env:
          IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }}

  integration-configs-db:
    needs: build
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout Repo
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Install Docker Client
        run: sudo ./.github/workflows/scripts/install-docker.sh
      - name: Download Docker Images Artifact
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          name: Docker Images
      - name: Extract Docker Images Archive
        run: tar -xvf images.tar -C /
      - name: Run Integration Configs Tests
        # Github Actions does not support TTY in their default runners yet
        run: |
          touch build-image/.uptodate
          MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations
          make BUILD_IMAGE=quay.io/cortexproject/build-image:master-0ddced051 TTY='' configs-integration-test

  deploy_website:
    needs: [build, test]
    if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
    runs-on: ubuntu-20.04
    container:
      image: quay.io/cortexproject/build-image:master-0ddced051
    steps:
      - name: Checkout Repo
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          # web-deploy script expects repo to be cloned with ssh for some commands to work
          ssh-key: ${{ secrets.WEBSITE_DEPLOY_SSH_PRIVATE_KEY }}
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Sym Link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Download Website Artifact
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          name: website public
          path: website/public
      - name: Setup SSH Keys and known_hosts for Github Authentication to Deploy Website
        run: |
          mkdir -p ~/.ssh
          ssh-keyscan github.com >> ~/.ssh/known_hosts
          ssh-agent -a $SSH_AUTH_SOCK > /dev/null
          ssh-add - <<< "${{ secrets.WEBSITE_DEPLOY_SSH_PRIVATE_KEY }}"
        env:
          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
        shell: bash
      - name: Deploy Website
        # SSH is used to authentricate with Github because web-deploy script uses git to checkout and push to gh-pages
        run: make BUILD_IN_CONTAINER=false web-deploy
        env:
          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
          GIT_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no"

  deploy:
    needs: [build, test, lint, integration, integration-configs-db]
    if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
    runs-on: ubuntu-20.04
    container:
      image: quay.io/cortexproject/build-image:master-0ddced051
    steps:
      - name: Checkout Repo
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Setup Git safe.directory
        run: |
          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
          git config --system --add safe.directory $GITHUB_WORKSPACE
      - name: Install Docker Client
        run: ./.github/workflows/scripts/install-docker.sh
      - name: Sym link Expected Path to Workspace
        run: |
          mkdir -p /go/src/github.com/cortexproject/cortex
          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
      - name: Download Docker Images Artifact
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          name: Docker Images
      - name: Extract Docker Images Archive
        run: tar -xvf images.tar -C /
      - name: Load Images
        run: |
          ln -s /tmp/images ./docker-images
          make BUILD_IN_CONTAINER=false load-images
      - name: Deploy
        run: |
          if [ -n "$DOCKER_REGISTRY_PASSWORD" ]; then
            docker login -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASSWORD"
          fi
          if [ -n "$QUAY_REGISTRY_PASSWORD" ]; then
            docker login -u "$QUAY_REGISTRY_USER" -p "$QUAY_REGISTRY_PASSWORD" quay.io;
          fi
          export IMAGE_TAG=$(make image-tag)
          ./push-images $NOQUAY
        env:
          DOCKER_REGISTRY_USER: ${{secrets.DOCKER_REGISTRY_USER}}
          DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
          QUAY_REGISTRY_USER: ${{secrets.QUAY_REGISTRY_USER}}
          QUAY_REGISTRY_PASSWORD: ${{secrets.QUAY_REGISTRY_PASSWORD}}
          NOQUAY: ${{secrets.NOQUAY}}