From 5ea2ffdfb1a000e6dd1f2309f76d73a311e549c5 Mon Sep 17 00:00:00 2001 From: Ndibe Raymond Olisaemeka Date: Thu, 21 Dec 2023 02:47:19 +0100 Subject: [PATCH] refactor github actions iteration=5 * use matrix strategy for similar jobs to save time * update version of third-party actions * use composite actions Issue: #1030 Signed-off-by: Ndibe Raymond Olisaemeka --- .../actions/docker_build_and_push/action.yml | 8 +- .github/actions/doctl_action/action.yaml | 22 +++++ .github/workflows/build_deploy_backend.yml | 6 +- .github/workflows/build_deploy_docs.yml | 2 +- .github/workflows/build_deploy_frontend.yml | 43 ++++------ .github/workflows/build_locust.yml | 85 ------------------- .github/workflows/create_destroy_test_vm.yaml | 31 ++----- .github/workflows/locust.yml | 72 ++++++++++++++++ .github/workflows/scale_backend.yml | 10 +-- 9 files changed, 129 insertions(+), 150 deletions(-) create mode 100644 .github/actions/doctl_action/action.yaml delete mode 100644 .github/workflows/build_locust.yml create mode 100644 .github/workflows/locust.yml diff --git a/.github/actions/docker_build_and_push/action.yml b/.github/actions/docker_build_and_push/action.yml index 766bacb44..93f7b9945 100644 --- a/.github/actions/docker_build_and_push/action.yml +++ b/.github/actions/docker_build_and_push/action.yml @@ -1,9 +1,9 @@ name: Docker build and push action inputs: - dockerhub_username: + username: required: true type: string - dockerhub_token: + token: required: true type: string context: @@ -31,8 +31,8 @@ runs: - name: Login to DockerHub uses: docker/login-action@v3 with: - username: ${{ inputs.dockerhub_username }} - password: ${{ inputs.dockerhub_token }} + username: ${{ inputs.username }} + password: ${{ inputs.token }} - name: Build and push id: docker_build diff --git a/.github/actions/doctl_action/action.yaml b/.github/actions/doctl_action/action.yaml new file mode 100644 index 000000000..8fdf5014e --- /dev/null +++ b/.github/actions/doctl_action/action.yaml @@ -0,0 +1,22 @@ +name: Doctl action +inputs: + token: + required: true + type: string + script: + required: true + type: string +outputs: + json_string: ${{ toJson(steps.script.outputs) }} + +runs: + using: "composite" + steps: + - name: Install doctl + uses: digitalocean/action-doctl@v2 + with: + token: ${{ inputs.token }} + + - name: execute script + id: script + run: ${{ inputs.script }} diff --git a/.github/workflows/build_deploy_backend.yml b/.github/workflows/build_deploy_backend.yml index f426a76e9..16c849726 100644 --- a/.github/workflows/build_deploy_backend.yml +++ b/.github/workflows/build_deploy_backend.yml @@ -24,8 +24,8 @@ jobs: - name: Build and push ${{ matrix.service }} uses: unstructuredstudio/zubhub/.github/actions/docker_build_and_push@master with: - dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} context: ./zubhub_backend/ file: ./zubhub_backend/compose/${{ matrix.service }}/prod/Dockerfile push: true @@ -46,7 +46,7 @@ jobs: source: "." target: "/home/zubhub-services/zubhub" - - uses: unsctructuredstudio/zubhub/.github/actions/ssh_action@master + - uses: unstructuredstudio/zubhub/.github/actions/ssh_action@master with: host: ${{ secrets.DO_BACKEND_HOST }} username: ${{ secrets.DO_BACKEND_USERNAME }} diff --git a/.github/workflows/build_deploy_docs.yml b/.github/workflows/build_deploy_docs.yml index bdab90f65..babbee87d 100644 --- a/.github/workflows/build_deploy_docs.yml +++ b/.github/workflows/build_deploy_docs.yml @@ -11,7 +11,7 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: unstructuredstudio/zubhub/.github/actions/checkout@master - uses: actions/setup-python@v5 with: python-version: "3.x" diff --git a/.github/workflows/build_deploy_frontend.yml b/.github/workflows/build_deploy_frontend.yml index ee047492d..a548d3528 100644 --- a/.github/workflows/build_deploy_frontend.yml +++ b/.github/workflows/build_deploy_frontend.yml @@ -16,51 +16,38 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + uses: unstructuredstudio/zubhub/.github/actions/checkout@master - - name: Checkout files - uses: actions/checkout@v4 - - - name: Build and push - id: docker_build - uses: docker/build-push-action@v5 + - name: Build and push frontend + uses: unstructuredstudio/zubhub/.github/actions/docker_build_and_push@master with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} context: ./zubhub_frontend/zubhub/ file: ./zubhub_frontend/zubhub/Dockerfile.prod push: true tags: unstructuredstudio/zubhub-frontend:latest - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} deploy: needs: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: unstructuredstudio/zubhub/.github/actions/checkout@master - - name: Copy file via scp - uses: appleboy/scp-action@master + - uses: unstructuredstudio/zubhub/.github/actions/scp_action@master with: - host: ${{ secrets.DO_FRONTEND_HOST }} - username: ${{ secrets.DO_FRONTEND_USERNAME }} + host: ${{ secrets.DO_BACKEND_HOST }} + username: ${{ secrets.DO_BACKEND_USERNAME }} key: ${{ secrets.DO_SSHKEY }} source: "." target: "/home/zubhub-frontend/zubhub" - - name: Executing remote command - uses: appleboy/ssh-action@master + - uses: unstructuredstudio/zubhub/.github/actions/ssh_action@master with: - host: ${{ secrets.DO_FRONTEND_HOST }} - username: ${{ secrets.DO_FRONTEND_USERNAME }} - port: ${{ secrets.DO_SSHPORT }} + host: ${{ secrets.DO_BACKEND_HOST }} + username: ${{ secrets.DO_BACKEND_USERNAME }} key: ${{ secrets.DO_SSHKEY }} - script: "cp /home/zubhub-frontend/zubhub/zubhub_frontend/zubhub/deploy_frontend.sh /home/zubhub-frontend/ && sudo bash /home/zubhub-frontend/deploy_frontend.sh" + script: | + cp /home/zubhub-frontend/zubhub/zubhub_frontend/zubhub/deploy_frontend.sh /home/zubhub-frontend/ + sudo bash /home/zubhub-frontend/deploy_frontend.sh" diff --git a/.github/workflows/build_locust.yml b/.github/workflows/build_locust.yml deleted file mode 100644 index 41ff94655..000000000 --- a/.github/workflows/build_locust.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Build/Deploy/Destroy Locust Service - -on: - push: - branches: - - master - paths: - - "locust/**" - - workflow_dispatch: - inputs: - action_type: - description: - "Do you want to build new locust container or deploy/destroy a locust service? type \ - 'build' to build new container, 'deploy' to deploy a locust service, 'destroy' to \ - destroy existing locust service. defaults to 'build' " - required: true - default: "build" - -jobs: - build: - if: ${{github.event_name == 'push' || github.event.inputs.action_type == 'build' }} - runs-on: ubuntu-latest - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Checkout files - uses: actions/checkout@v4 - - - name: Build and push locust - id: docker_build_locust - uses: docker/build-push-action@v5 - with: - context: ./locust/ - file: ./locust/Dockerfile - push: true - tags: unstructuredstudio/zubhub-services_locust:latest - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - - deploy: - if: ${{ github.event.inputs.action_type == 'deploy' }} - runs-on: ubuntu-latest - steps: - - name: Install doctl - uses: digitalocean/action-doctl@v2 - with: - token: ${{ secrets.DO_ACCESS_TOKEN }} - - - name: Create new droplet - run: | - doctl compute droplet create locust --image \ - ${{ secrets.SOURCE_SNAPSHOT_ID }} --tag-name zubhub-locust --size s-1vcpu-1gb \ - --region nyc1 --enable-monitoring --ssh-keys ${{ secrets.DO_PUBLIC_SSHKEY_FP }} --wait - sleep 30s - echo "NEW_DROPLET_IP=$(doctl compute droplet get locust \ - --template "{{(index .Networks.V4 1).IPAddress}}")" >> $GITHUB_ENV - - - name: Deploy locust - uses: appleboy/ssh-action@master - with: - HOST: ${{ env.NEW_DROPLET_IP }} - USERNAME: ${{ secrets.DO_BACKEND_USERNAME }} - KEY: ${{ secrets.DO_SSHKEY }} - script: "docker run -d -p 8089:8089 unstructuredstudio/zubhub-services_locust:latest -f /mnt/locust/locustfile.py" - - destroy: - if: ${{ github.event.inputs.action_type == 'destroy' }} - runs-on: ubuntu-latest - steps: - - name: Install doctl - uses: digitalocean/action-doctl@v2 - with: - token: ${{ secrets.DO_ACCESS_TOKEN }} - - - name: Destroy Target Droplet - run: "doctl compute droplet delete -f locust" diff --git a/.github/workflows/create_destroy_test_vm.yaml b/.github/workflows/create_destroy_test_vm.yaml index 30c92dfa6..5c050c2da 100644 --- a/.github/workflows/create_destroy_test_vm.yaml +++ b/.github/workflows/create_destroy_test_vm.yaml @@ -46,8 +46,7 @@ jobs: # See https://stackoverflow.com/questions/75873833/how-to-protect-github-secrets-in-pull-request-actions-from-malicious-pull-reques # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch. - - name: Checkout source branch - uses: actions/checkout@v4 + uses: unstructuredstudio/zubhub/.github/actions/checkout@master with: ref: ${{ env.CHECKOUT_COMMIT_REF }} repository: ${{ env.CHECKOUT_REPO }} @@ -94,8 +93,7 @@ jobs: doctl compute domain records create unstructured.studio --record-type A --record-name \ $MEDIA_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600 - - name: Copy file via scp - uses: appleboy/scp-action@master + uses: unstructuredstudio/zubhub/.github/actions/scp_action@master with: host: ${{env.NEW_DROPLET_IP}} username: ${{ secrets.DO_BACKEND_USERNAME }} @@ -147,8 +145,7 @@ jobs: # See https://stackoverflow.com/questions/75873833/how-to-protect-github-secrets-in-pull-request-actions-from-malicious-pull-reques # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch. - - name: Checkout source branch - uses: actions/checkout@v4 + uses: unstructuredstudio/zubhub/.github/actions/checkout@master with: ref: ${{ env.CHECKOUT_COMMIT_REF }} repository: ${{ env.CHECKOUT_REPO }} @@ -174,22 +171,11 @@ jobs: REACT_APP_DEV_VIDEO_UPLOAD_PRESET_NAME=dev_video_upload_preset EOF - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push ${{ matrix.service }} - id: docker_build - uses: docker/build-push-action@v5 + uses: unstructuredstudio/zubhub/.github/actions/docker_build_and_push@master with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} # for frontend, context is ./zubhub_frontend/zubhub/ # for backend services, context is ./zubhub_backend/ context: ./zubhub_${{ matrix.service == 'frontend' && 'frontend/zubhub' || 'backend' }}/ @@ -198,9 +184,6 @@ jobs: file: ./zubhub_${{ matrix.service == 'frontend' && 'frontend/zubhub' || format('backend/compose/{0}', matrix.service) }}/${{ matrix.service != 'frontend' && 'prod/' || '' }}Dockerfile${{ matrix.service == 'frontend' && '.prod' || '' }} push: true tags: unstructuredstudio/zubhub-test_${{ matrix.service }}:latest - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} ################################################################################## ################################################################################# @@ -221,7 +204,7 @@ jobs: echo "MEDIA_DOMAIN=${{ needs.create_test_vm.outputs.MEDIA_DOMAIN }}" >> $GITHUB_ENV - name: Executing remote command - uses: appleboy/ssh-action@master + uses: unstructuredstudio/zubhub/.github/actions/ssh_action@master with: host: ${{env.NEW_DROPLET_IP}} username: ${{ secrets.DO_BACKEND_USERNAME }} diff --git a/.github/workflows/locust.yml b/.github/workflows/locust.yml new file mode 100644 index 000000000..da618e4f2 --- /dev/null +++ b/.github/workflows/locust.yml @@ -0,0 +1,72 @@ +name: Build/Deploy/Destroy Locust Service + +on: + push: + branches: + - master + paths: + - "locust/**" + + workflow_dispatch: + inputs: + action_type: + description: + "Do you want to build new locust container or deploy/destroy a locust service? type \ + 'build' to build new container, 'deploy' to deploy a locust service, 'destroy' to \ + destroy existing locust service. defaults to 'build' " + required: true + default: "deploy" + +jobs: + build: + if: ${{github.event_name == 'push' || github.event.inputs.action_type == 'build' }} + runs-on: ubuntu-latest + steps: + - uses: unstructuredstudio/zubhub/.github/actions/checkout@master + + - name: Build and push locust + uses: unstructuredstudio/zubhub/.github/actions/docker_build_and_push@master + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + context: ./locust/ + file: ./locust/Dockerfile + push: true + tags: unstructuredstudio/zubhub-services_locust:latest + + deploy: + if: ${{ github.event.inputs.action_type == 'deploy' }} + runs-on: ubuntu-latest + steps: + - name: Create new DO droplet + id: create_droplet + uses: unsctructuredstudio/zubhub/.github/actions/doctl@master + with: + token: ${{ secrets.DO_ACCESS_TOKEN }} + script: | + doctl compute droplet create locust --image \ + ${{ secrets.SOURCE_SNAPSHOT_ID }} --tag-name zubhub-locust --size s-1vcpu-1gb \ + --region nyc1 --enable-monitoring --ssh-keys ${{ secrets.DO_PUBLIC_SSHKEY_FP }} --wait + sleep 30s + echo "NEW_DROPLET_IP=$(doctl compute droplet get locust \ + --template "{{(index .Networks.V4 1).IPAddress}}")" >> $GITHUB_OUTPUT + + - name: Deploy locust + uses: unstructuredstudio/zubhub/.github/actions/ssh_action@master + with: + host: ${{ fromJson(steps.create_droplet.outputs.json_string).NEW_DROPLET_IP }} + username: ${{ secrets.DO_BACKEND_USERNAME }} + key: ${{ secrets.DO_SSHKEY }} + script: | + docker run -d -p 8089:8089 unstructuredstudio/zubhub-services_locust:latest \ + -f /mnt/locust/locustfile.py + + destroy: + if: ${{ github.event.inputs.action_type == 'destroy' }} + runs-on: ubuntu-latest + steps: + - name: Destroy Target Droplet + uses: unstructuredstudio/zubhub/.github/actions/doctl@master + with: + token: ${{ secrets.DO_ACCESS_TOKEN }} + script: doctl compute droplet delete -f locust diff --git a/.github/workflows/scale_backend.yml b/.github/workflows/scale_backend.yml index b2fb67539..615b84a31 100644 --- a/.github/workflows/scale_backend.yml +++ b/.github/workflows/scale_backend.yml @@ -31,7 +31,7 @@ jobs: --template "{{(index .Networks.V4 1).IPAddress}}")" >> $GITHUB_ENV - name: Connect new droplet to swarm - uses: appleboy/ssh-action@master + uses: unstructuredstudio/zubhub/.github/actions/ssh_action@master with: HOST: ${{ env.NEW_DROPLET_IP }} USERNAME: ${{ secrets.DO_BACKEND_USERNAME }} @@ -39,7 +39,7 @@ jobs: script: "docker swarm join --token ${{secrets.SWARM_WORKER_JOIN_TOKEN}} ${{secrets.SWARM_MASTER_HOST_AND_PORT}};sleep 10" - name: Scale up deployment - uses: appleboy/ssh-action@master + uses: unstructuredstudio/zubhub/.github/actions/ssh_action@master with: HOST: ${{ secrets.DO_BACKEND_HOST }} USERNAME: ${{ secrets.DO_BACKEND_USERNAME }} @@ -65,7 +65,7 @@ jobs: - name: Scale down deployment if: ${{ env.TARGET_DROPLETS_COUNT > 0 }} - uses: appleboy/ssh-action@master + uses: unstructuredstudio/zubhub/.github/actions/ssh_action@master with: HOST: ${{ secrets.DO_BACKEND_HOST }} USERNAME: ${{ secrets.DO_BACKEND_USERNAME }} @@ -74,7 +74,7 @@ jobs: - name: Disconnect Target Droplet From Swarm if: ${{ env.TARGET_DROPLETS_COUNT > 0 }} - uses: appleboy/ssh-action@master + uses: unstructuredstudio/zubhub/.github/actions/ssh_action@master with: HOST: ${{ env.TARGET_DROPLET_IP }} USERNAME: ${{ secrets.DO_BACKEND_USERNAME }} @@ -89,7 +89,7 @@ jobs: - name: Remove Target Droplet From Node List if: ${{ env.TARGET_DROPLETS_COUNT > 0 }} - uses: appleboy/ssh-action@master + uses: unstucturedstudio/zubhub/.github/actions/ssh_action@master with: HOST: ${{ secrets.DO_BACKEND_HOST }} USERNAME: ${{ secrets.DO_BACKEND_USERNAME }}