From 4e187fc45750249ad2d030b156e4276bf7ef3ed2 Mon Sep 17 00:00:00 2001 From: Jack Waller Date: Tue, 3 Dec 2024 14:32:43 +1100 Subject: [PATCH 1/9] chore: add new piplines --- .github/workflows/mainnet-beta.yml | 68 ++++++++++++++++++++++++++++++ .github/workflows/master.yml | 68 ++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 .github/workflows/mainnet-beta.yml create mode 100644 .github/workflows/master.yml diff --git a/.github/workflows/mainnet-beta.yml b/.github/workflows/mainnet-beta.yml new file mode 100644 index 0000000..ead2e13 --- /dev/null +++ b/.github/workflows/mainnet-beta.yml @@ -0,0 +1,68 @@ +name: Build Image And Deploy + +on: + push: + branches: [mainnet-beta] + +jobs: + build: + runs-on: ubicloud + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@master + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PROD }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_PROD }} + aws-region: ${{ secrets.EKS_PROD_REGION }} + + - name: Log in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Docker build + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: usermap-server + IMAGE_TAG: ${{ github.sha }} + BRANCH_NAME: ${{ github.ref_name }} + run: | + docker build \ + -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ + -t $ECR_REGISTRY/$ECR_REPOSITORY:latest-$BRANCH_NAME \ + -f ./Dockerfile \ + . + + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest-$BRANCH_NAME + + deploy: + runs-on: ubicloud + needs: [build] + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@master + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PROD }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_PROD }} + aws-region: ${{ secrets.EKS_PROD_REGION }} + + - name: Install kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'v1.30.0' + + - name: Configure AWS EKS Credentials + run: aws eks update-kubeconfig --name ${{ secrets.EKS_PROD_CLUSTER_NAME }} --region ${{ secrets.EKS_PROD_REGION }} --role-arn ${{ secrets.EKS_PROD_DEPLOY_ROLE }} + + - name: Restart deployment + env: + BRANCH_NAME: ${{ github.ref_name }} + run: | + kubectl rollout restart -n $BRANCH_NAME deployment/usermap-publisher-app + kubectl rollout restart -n $BRANCH_NAME deployment/usermap-publisher-app-local \ No newline at end of file diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml new file mode 100644 index 0000000..cda872b --- /dev/null +++ b/.github/workflows/master.yml @@ -0,0 +1,68 @@ +name: Build Image And Deploy + +on: + push: + branches: [master, staging] + +jobs: + build: + runs-on: ubicloud + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@master + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_NON_PROD }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_NON_PROD }} + aws-region: ${{ secrets.EKS_NON_PROD_REGION }} + + - name: Log in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Docker build + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: usermap-server + IMAGE_TAG: ${{ github.sha }} + BRANCH_NAME: ${{ github.ref_name }} + run: | + docker build \ + -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ + -t $ECR_REGISTRY/$ECR_REPOSITORY:latest-$BRANCH_NAME \ + -f ./Dockerfile \ + . + + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest-$BRANCH_NAME + + deploy: + runs-on: ubicloud + needs: [build] + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@master + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_NON_PROD }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_NON_PROD }} + aws-region: ${{ secrets.EKS_NON_PROD_REGION }} + + - name: Install kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'v1.30.0' + + - name: Configure AWS EKS Credentials + run: aws eks update-kubeconfig --name ${{ secrets.EKS_NON_PROD_CLUSTER_NAME }} --region ${{ secrets.EKS_NON_PROD_REGION }} --role-arn ${{ secrets.EKS_NON_PROD_DEPLOY_ROLE }} + + - name: Restart deployment + env: + BRANCH_NAME: ${{ github.ref_name }} + run: | + kubectl rollout restart -n $BRANCH_NAME deployment/usermap-publisher-app + kubectl rollout restart -n $BRANCH_NAME deployment/usermap-publisher-app-local \ No newline at end of file From c55469a80efdc369fa8b3acda63e12870cade892 Mon Sep 17 00:00:00 2001 From: Jack Waller Date: Tue, 3 Dec 2024 15:11:54 +1100 Subject: [PATCH 2/9] chore: checkout with submodules --- .github/workflows/mainnet-beta.yml | 2 ++ .github/workflows/master.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/mainnet-beta.yml b/.github/workflows/mainnet-beta.yml index ead2e13..8f97479 100644 --- a/.github/workflows/mainnet-beta.yml +++ b/.github/workflows/mainnet-beta.yml @@ -10,6 +10,8 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v3 + with: + submodules: 'recursive' - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@master diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index cda872b..9a193e4 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -10,6 +10,8 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v3 + with: + submodules: 'recursive' - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@master From cbaac1f43c22298ca86360abcae22f92327703f4 Mon Sep 17 00:00:00 2001 From: Jack Waller Date: Tue, 3 Dec 2024 15:25:56 +1100 Subject: [PATCH 3/9] test: caching docker layer --- .github/workflows/master.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 9a193e4..424694e 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -24,21 +24,27 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - - name: Docker build + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push + uses: docker/build-push-action@v4 env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: usermap-server IMAGE_TAG: ${{ github.sha }} BRANCH_NAME: ${{ github.ref_name }} - run: | - docker build \ - -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ - -t $ECR_REGISTRY/$ECR_REPOSITORY:latest-$BRANCH_NAME \ - -f ./Dockerfile \ - . - - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest-$BRANCH_NAME + with: + context: . + push: true + tags: | + ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest-${{ env.BRANCH_NAME }}-amd64 + cache-from: | + type=registry,ref=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:cache-${{ env.BRANCH_NAME }} + cache-to: | + type=registry,ref=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:cache-${{ env.BRANCH_NAME }},mode=max deploy: runs-on: ubicloud From 10275e56a359ebef05286cf3a5ad21e05a3adfa0 Mon Sep 17 00:00:00 2001 From: Jack Waller Date: Tue, 3 Dec 2024 15:35:41 +1100 Subject: [PATCH 4/9] test: use gha cache --- .github/workflows/mainnet-beta.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/mainnet-beta.yml b/.github/workflows/mainnet-beta.yml index 8f97479..ff85578 100644 --- a/.github/workflows/mainnet-beta.yml +++ b/.github/workflows/mainnet-beta.yml @@ -24,21 +24,24 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - - name: Docker build + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push + uses: docker/build-push-action@v4 env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: usermap-server IMAGE_TAG: ${{ github.sha }} BRANCH_NAME: ${{ github.ref_name }} - run: | - docker build \ - -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ - -t $ECR_REGISTRY/$ECR_REPOSITORY:latest-$BRANCH_NAME \ - -f ./Dockerfile \ - . - - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest-$BRANCH_NAME + with: + context: . + push: true + tags: | + ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest-${{ env.BRANCH_NAME }}-amd64 + cache-from: type=gha + cache-to: type=gha,mode=max deploy: runs-on: ubicloud From b81f0a2b8c42465e18018e5cffe0660d9fe835e2 Mon Sep 17 00:00:00 2001 From: Jack Waller Date: Tue, 3 Dec 2024 15:36:05 +1100 Subject: [PATCH 5/9] chore: use gha cache --- .github/workflows/master.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 424694e..2e90645 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -41,10 +41,8 @@ jobs: tags: | ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest-${{ env.BRANCH_NAME }}-amd64 - cache-from: | - type=registry,ref=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:cache-${{ env.BRANCH_NAME }} - cache-to: | - type=registry,ref=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:cache-${{ env.BRANCH_NAME }},mode=max + cache-from: type=gha + cache-to: type=gha,mode=max deploy: runs-on: ubicloud From 0626c6d00484d62971e653337bc454ebde3e3b9c Mon Sep 17 00:00:00 2001 From: Jack Waller Date: Tue, 3 Dec 2024 15:44:49 +1100 Subject: [PATCH 6/9] chore: update push version --- .github/workflows/mainnet-beta.yml | 4 ++-- .github/workflows/master.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/mainnet-beta.yml b/.github/workflows/mainnet-beta.yml index ff85578..fa39f0b 100644 --- a/.github/workflows/mainnet-beta.yml +++ b/.github/workflows/mainnet-beta.yml @@ -25,10 +25,10 @@ jobs: uses: aws-actions/amazon-ecr-login@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: usermap-server diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 2e90645..ab7b7e9 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -26,10 +26,10 @@ jobs: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: usermap-server From 4b5e31186077b874ca5fbdb8e80e96b732d5daa5 Mon Sep 17 00:00:00 2001 From: Jack Waller Date: Tue, 3 Dec 2024 16:13:22 +1100 Subject: [PATCH 7/9] chore: remove cache for now --- .github/workflows/mainnet-beta.yml | 5 ----- .github/workflows/master.yml | 8 +------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/mainnet-beta.yml b/.github/workflows/mainnet-beta.yml index fa39f0b..4a577d4 100644 --- a/.github/workflows/mainnet-beta.yml +++ b/.github/workflows/mainnet-beta.yml @@ -24,9 +24,6 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Build and push uses: docker/build-push-action@v6 env: @@ -40,8 +37,6 @@ jobs: tags: | ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest-${{ env.BRANCH_NAME }}-amd64 - cache-from: type=gha - cache-to: type=gha,mode=max deploy: runs-on: ubicloud diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index ab7b7e9..185ea7c 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -23,11 +23,7 @@ jobs: - name: Log in to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - + - name: Build and push uses: docker/build-push-action@v6 env: @@ -41,8 +37,6 @@ jobs: tags: | ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest-${{ env.BRANCH_NAME }}-amd64 - cache-from: type=gha - cache-to: type=gha,mode=max deploy: runs-on: ubicloud From 0ffe50417c08b927d8ac35224f63fa749e4249bf Mon Sep 17 00:00:00 2001 From: jordy25519 Date: Wed, 4 Dec 2024 14:04:18 +0800 Subject: [PATCH 8/9] small tidy ups to gh pipeline --- .github/workflows/mainnet-beta.yml | 5 +---- .github/workflows/master.yml | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/mainnet-beta.yml b/.github/workflows/mainnet-beta.yml index 4a577d4..14bdf10 100644 --- a/.github/workflows/mainnet-beta.yml +++ b/.github/workflows/mainnet-beta.yml @@ -11,7 +11,7 @@ jobs: - name: Checkout Code uses: actions/checkout@v3 with: - submodules: 'recursive' + submodules: recursive - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@master @@ -42,9 +42,6 @@ jobs: runs-on: ubicloud needs: [build] steps: - - name: Checkout Code - uses: actions/checkout@v3 - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@master with: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 185ea7c..84c414d 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -11,7 +11,7 @@ jobs: - name: Checkout Code uses: actions/checkout@v3 with: - submodules: 'recursive' + submodules: recursive - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@master @@ -42,9 +42,6 @@ jobs: runs-on: ubicloud needs: [build] steps: - - name: Checkout Code - uses: actions/checkout@v3 - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@master with: From 2c4c5ab5a6fb8acd24ad46b290f377846c85e0b3 Mon Sep 17 00:00:00 2001 From: Jack Waller Date: Thu, 5 Dec 2024 12:19:25 +1100 Subject: [PATCH 9/9] chore: add branch names to tag --- .github/workflows/mainnet-beta.yml | 2 +- .github/workflows/master.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mainnet-beta.yml b/.github/workflows/mainnet-beta.yml index 14bdf10..ecfa0c8 100644 --- a/.github/workflows/mainnet-beta.yml +++ b/.github/workflows/mainnet-beta.yml @@ -35,7 +35,7 @@ jobs: context: . push: true tags: | - ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}-${{ env.BRANCH_NAME }}-amd64 ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest-${{ env.BRANCH_NAME }}-amd64 deploy: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 84c414d..0ed6f04 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -35,7 +35,7 @@ jobs: context: . push: true tags: | - ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}-${{ env.BRANCH_NAME }}-amd64 ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest-${{ env.BRANCH_NAME }}-amd64 deploy: