Migrate CI to github-actions #9
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - deploy/* | |
| pull_request: | |
| concurrency: | |
| group: ${{ github.ref }}-x | |
| cancel-in-progress: true | |
| env: | |
| CACHE_REGISTRY: 127.0.0.1:5000 | |
| AWS_REGION: ap-southeast-2 | |
| PROJECT: madewithwagtail | |
| jobs: | |
| static-analysis: | |
| container: ghcr.io/springload/python-static-analysis:latest | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| # Infra code checks | |
| - name: Check ssm.ejson format | |
| run: python -m json.tool infra/ssm.ejson > /dev/null | |
| - name: check ssm.ejson encrypted | |
| run: | | |
| wget -O- https://github.com/Shopify/ejson/releases/download/v1.3.3/ejson_1.3.3_linux_amd64.tar.gz| tar xzv -C /usr/bin/ -f - ejson | |
| cp infra/ssm.ejson /tmp/original-ssm.ejson | |
| ejson encrypt infra/ssm.ejson | |
| diff infra/ssm.ejson /tmp/original-ssm.ejson | |
| # Python code checks | |
| - name: Check python imports formatted | |
| run: isort . --check-only --diff --quiet | |
| - name: Python format check | |
| run: black --check --diff . | |
| - name: Python code style checks | |
| run: flakeheaven lint | |
| - name: Python bandit check | |
| run: bandit -r . -c pyproject.toml | |
| - name: Python safety check | |
| run: safety check -r requirements/constraints.txt | |
| # This job uses Skopeo to check if we already have | |
| # the image with the same git commit hash in the ECR | |
| # If it does, it just copies it and skips the actual build. | |
| retag-images: | |
| if: startsWith(github.ref, 'refs/heads/deploy/') | |
| needs: | |
| - static-analysis | |
| outputs: | |
| skip_build: ${{ steps.retag.outputs.skip_build }} | |
| container: docker:stable-git | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| - name: Retag images with Skopeo | |
| id: retag | |
| run: | | |
| apk add skopeo | |
| # we tag images with short commit which is the first 8 chars | |
| # of the commit | |
| SHORT_COMMIT=$(echo $GITHUB_SHA | cut -c -8) | |
| ENVIRONMENT=$(basename $GITHUB_REF) | |
| # even with "set -e" it won't fail | |
| # as it's handled in if | |
| if printf "app\nhttpd" | xargs -I{} skopeo inspect docker://${{ steps.login-ecr.outputs.registry }}/${{ env.PROJECT }}-{}:common-${SHORT_COMMIT}; then | |
| # retag the image by "copying" it | |
| printf "app\nhttpd" | xargs -I{} skopeo copy \ | |
| docker://${{ steps.login-ecr.outputs.registry }}/${{ env.PROJECT }}-{}:common-${SHORT_COMMIT} \ | |
| docker://${{ steps.login-ecr.outputs.registry }}/${{ env.PROJECT }}-{}:${ENVIRONMENT}-${SHORT_COMMIT} | |
| echo "::set-output name=skip_build::true" | |
| echo "Skipping the next build" | |
| else | |
| echo "::set-output name=skip_build::false" | |
| fi | |
| # This job heavily relies on Docker layer caching to make it as fast as possible | |
| # It builds the app-test stage first, and tests it, | |
| # and only after that builds the rest of the stuff and pushes it to ECR | |
| # in one go. | |
| build-and-test: | |
| # builds the image if the previous job didn't fail and didn't indicate | |
| # that this one should be skipped | |
| if: ${{ !failure() && (needs.retag-images.outputs.skip_build!='true')}} | |
| needs: [retag-images, static-analysis] | |
| container: docker:stable-git | |
| runs-on: ubuntu-latest | |
| services: | |
| registry: | |
| image: registry:2.7.1 | |
| env: | |
| REGISTRY_STORAGE: s3 | |
| REGISTRY_STORAGE_S3_REGION: ${{ env.AWS_REGION }} | |
| REGISTRY_STORAGE_S3_BUCKET: ${{ secrets.REGISTRY_BUCKET_NAME }} | |
| REGISTRY_STORAGE_S3_ACCESSKEY: ${{ secrets.REGISTRY_AWS_ACCESS_KEY_ID }} | |
| REGISTRY_STORAGE_S3_SECRETKEY: ${{ secrets.REGISTRY_AWS_SECRET_ACCESS_KEY }} | |
| ports: | |
| - 127.0.0.1:5000:5000/tcp | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v1 | |
| with: | |
| version: v0.6.3 | |
| # needs to use host network to access registry at 127.0.0.1 | |
| driver-opts: network=host | |
| - name: Set dynamic env vars | |
| run: | | |
| docker version | |
| SHORT_COMMIT=$(echo $GITHUB_SHA | cut -c -8) | |
| echo "VERSION=${SHORT_COMMIT}" >> $GITHUB_ENV | |
| echo "DATABASE_PASSWORD=$( head -c 24 /dev/urandom | xxd -p | tr -d '\n ')" >> $GITHUB_ENV | |
| echo "ENVIRONMENT=$(basename $GITHUB_REF)" >> $GITHUB_ENV | |
| - name: Warm up buildx cache by building base | |
| uses: docker/bake-action@master | |
| with: | |
| files: ./docker-bake.hcl | |
| targets: base | |
| - name: Build app-test | |
| uses: docker/bake-action@master | |
| with: | |
| files: docker-bake.hcl | |
| targets: app-test | |
| load: true | |
| - name: Test app | |
| run: | | |
| # there's some limitation on the hostname length | |
| # so we'll just limit it to 12 symbols using "cut" | |
| # for cache and database URLs | |
| DB=$(docker run --rm \ | |
| -ePOSTGRES_DB=${{ env.PROJECT }}_test \ | |
| -ePOSTGRES_PASSWORD=$DATABASE_PASSWORD \ | |
| -d postgres:13-alpine | cut -c -12) | |
| CACHE=$(docker run --rm -d redis:3.2-alpine | cut -c -12) | |
| docker run --rm -i \ | |
| --link=$DB --link=$CACHE \ | |
| -eDATABASE_URL=postgres://postgres:$DATABASE_PASSWORD@$DB/${{ env.PROJECT }}_test \ | |
| -eENVIRONMENT=test \ | |
| -eDJANGO_SETTINGS_MODULE=app.settings.test \ | |
| -eCACHE_URL=redis://$CACHE:6379/0 \ | |
| -eTASK_QUEUE_URL=redis://$CACHE:6379/1 \ | |
| ${{ env.PROJECT }}/app-test:${{ env.VERSION }} | |
| docker stop $DB | |
| docker stop $CACHE | |
| - name: Login to Amazon ECR | |
| if: startsWith(github.ref, 'refs/heads/deploy/') | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| - name: Build all other app parts and push to ECR | |
| if: startsWith(github.ref, 'refs/heads/deploy/') | |
| env: | |
| REGISTRY: ${{ steps.login-ecr.outputs.registry }} | |
| uses: docker/bake-action@master | |
| with: | |
| files: docker-bake.hcl | |
| targets: default | |
| # makes it push to the registry | |
| push: true | |
| deploy: | |
| needs: [build-and-test, retag-images] | |
| if: startsWith(github.ref, 'refs/heads/deploy/') && !failure() | |
| container: docker:stable-git | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Download ecs-tool | |
| run: | | |
| wget -O ecs-tool.tar.gz https://github.com/springload/ecs-tool/releases/download/1.9.0/ecs-tool_1.9.0_linux_amd64.tar.gz && tar -C /usr/bin -xvf ecs-tool.tar.gz ecs-tool | |
| - name: Deploy app | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| EJSON_PRIVATE: ${{ secrets.EJSON_PRIVATE }} | |
| run: |- | |
| set -eu | |
| ENVIRONMENT=$(basename "${GITHUB_REF}") | |
| VERSION=$(echo $GITHUB_SHA | cut -c -8) | |
| echo "updating the ssm parameter with ejson configuration" | |
| ecs-tool -p "" -e "${ENVIRONMENT}" ejson -f infra/ssm.ejson | |
| ecs-tool -p "" -e "${ENVIRONMENT}" run --image_tag "${ENVIRONMENT}-${VERSION}" -- ./deploy.sh | |
| ecs-tool -p "" -e "${ENVIRONMENT}" deploy --image_tag "${ENVIRONMENT}-${VERSION}" | |
| test-with-zap: | |
| # have to have the if here as this job depends on another, which | |
| # itself depends on a job that can be skipped | |
| if: startsWith(github.ref, 'refs/heads/deploy/') && !failure() | |
| env: | |
| BASIC_AUTH: ${{ secrets.BASIC_AUTH }} | |
| needs: deploy | |
| container: owasp/zap2docker-weekly | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Test the website with zap | |
| run: | | |
| set -e | |
| ENVIRONMENT=`basename "${GITHUB_REF}"` | |
| if [ "${ENVIRONMENT}" = "production" ]; then | |
| endpoint=https://madewithwagtail.org/ | |
| else | |
| endpoint=https://${PROJECT}-${ENVIRONMENT}.springload.nz | |
| fi | |
| zap-baseline.py \ | |
| -z "-config replacer.full_list(0).description=auth1 \ | |
| -config replacer.full_list(0).enabled=true \ | |
| -config replacer.full_list(0).matchtype=REQ_HEADER \ | |
| -config replacer.full_list(0).matchstr=Authorization \ | |
| -config replacer.full_list(0).regex=false \ | |
| -config replacer.full_list(0).replacement=Basic\ ${BASIC_AUTH}" \ | |
| --autooff -I -t $endpoint |