From e1b1529ba19dc736f643d44b03f97ca79e1a87f6 Mon Sep 17 00:00:00 2001 From: mahesh-naxa Date: Fri, 15 Nov 2024 13:54:14 +0545 Subject: [PATCH] infra: TODO: add a dummy deployment workflow, --- .github/workflows/build_and_deploy.yml | 35 ++++--- .github/workflows/remote_deploy_compose.yml | 100 ++++++++++++++++++++ .github/workflows/test_compose.yml | 8 +- 3 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/remote_deploy_compose.yml diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index c52339cb10..c77121e2fc 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -35,6 +35,8 @@ jobs: frontend-build: uses: naxa-developers/tasking-manager/.github/workflows/frontend-build.yml@ci-gh-workflows secrets: inherit + needs: + - frontend-test with: node-version: 16.x context: ./frontend @@ -54,16 +56,28 @@ jobs: with: name: ${{ needs.frontend-build.outputs.artifact-name }} path: ./build - + #TODO: Handle Auth and upload to respective s3 bucket & do a cloudfront invalidation - name: Debug check files run: | ls -alh ls -alh build + backend-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@2.0.5 + with: + context: . + build_target: prod + image_name: ghcr.io/${{ github.repository }}/backend:$GITHUB_SHA + dockerfile: Dockerfile + scan_image: false + secrets: inherit + backend-test: + needs: + - backend-build uses: naxa-developers/tasking-manager/.github/workflows/test_compose.yml@ci-gh-workflows with: - image_name: ghcr.io/${{ github.repository }}/backend + image_name: ghcr.io/${{ github.repository }}/backend:$GITHUB_SHA pre_command: | docker compose --file docker-compose.yml up -d sleep 15 @@ -73,31 +87,22 @@ jobs: # TODO: Compose command to be updated once dev have test setup running. For now just a simple curl command. compose_command: curl -i http://localhost:5000/api/v2/ tag_override: ci-${{ github.ref_name }} - # coverage: true + run_tests: true + coverage: false example_env_file_path: example.env env_file_path: tasking-manager.env secrets: inherit - - backend-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@2.0.5 - with: - context: . - build_target: prod - image_name: ghcr.io/${{ github.repository }}/backend - dockerfile: Dockerfile - scan_image: false - secrets: inherit - + backup-database: runs-on: ubuntu-latest needs: - - backend-build - backend-test name: Backup Database before deployment steps: - name: Create Snapshot run: echo " TODO Database backup Steps; See CircleCI" + # See https://app.circleci.com/pipelines/github/hotosm/tasking-manager/11487/workflows/e98ba643-5812-4b2a-a09f-cc499285b3cc/jobs/23599 #TODO: Check circleci for references backend_deploy_to_vm: diff --git a/.github/workflows/remote_deploy_compose.yml b/.github/workflows/remote_deploy_compose.yml new file mode 100644 index 0000000000..af699f89bc --- /dev/null +++ b/.github/workflows/remote_deploy_compose.yml @@ -0,0 +1,100 @@ +# Note: variables: SSH_HOST and SSH_USER must be set for your environment. +# Note: secrets: SSH_PRIVATE_KEY must be set for your environment. + +name: Remote Deploy (Compose) + +on: + workflow_call: + inputs: + environment: + description: "The Github environment to get variables from. Default repository vars." + required: false + type: string + docker_compose_file: + description: "Path to docker compose file to deploy." + required: true + type: string + example_env_file_path: + description: "Path to example dotenv file to substitute variables for." + type: string + default: .env.example + env_file_path: + description: "Path to write dotenv file" + type: string + default: .env + +jobs: + remote-deploy: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Vars and Secrets to Env + env: + GIT_BRANCH: ${{ github.ref_name }} + VARS_CONTEXT: ${{ toJson(vars) }} + SECRETS_CONTEXT: ${{ toJson(secrets) }} + run: | + # Random delimeter string for security + delim=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + + # Parse JSON with multiline strings, using delimeter (Github specific) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$delim\n\(.value)\n$delim\n\""; } + + # Set vars to env for next step + echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV + echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> $GITHUB_ENV + + # Set VARS_CONTEXT if not null + if [ "${VARS_CONTEXT}" != "null" ]; then + echo "${VARS_CONTEXT}" | to_envs >> $GITHUB_ENV + fi + + # Set SECRETS_CONTEXT if not null + if [ "${SECRETS_CONTEXT}" != "null" ]; then + echo "${SECRETS_CONTEXT}" | to_envs >> $GITHUB_ENV + fi + + - name: Create .env file + env: + EXAMPLE_DOTENV: ${{ inputs.example_env_file_path }} + run: | + echo "Checking if ${EXAMPLE_DOTENV} exists" + if [ -f ${EXAMPLE_DOTENV} ]; then + # Get a8m/envsubst (required for default vals syntax ${VAR:-default}) + echo "Downloading envsubst" + curl -L https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-`uname -s`-`uname -m` -o envsubst + if [ $? -ne 0 ]; then + echo "Failed to download envsubst" + exit 1 + fi + chmod +x envsubst + echo "Substituting variables from ${EXAMPLE_DOTENV} --> ${{ inputs.env_file_path }}" + ./envsubst < "${EXAMPLE_DOTENV}" > ${{ inputs.env_file_path }} + else + echo "${EXAMPLE_DOTENV} not found, creating empty ${{ inputs.env_file_path }}" + touch ${{ inputs.env_file_path }} + fi + + echo "GIT_BRANCH=${GIT_BRANCH}" >> ${{ inputs.env_file_path }} + echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> ${{ inputs.env_file_path }} + + # TODO: Add step to force new deployment here: also update image_tag accordingly in terraform vars. + # - uses: webfactory/ssh-agent@v0.8.0 + # with: + # ssh-private-key: "${{ secrets.SSH_PRIVATE_KEY }}" + + # - name: Add host keys to known_hosts + # run: | + # ssh-keyscan "${{ vars.SSH_HOST }}" >> ~/.ssh/known_hosts + + # - name: Deploy + # run: | + # docker compose --file ${{ inputs.docker_compose_file }} pull + # docker compose --file ${{ inputs.docker_compose_file }} up \ + # --detach --remove-orphans --force-recreate + # env: + # DOCKER_HOST: "ssh://${{ vars.SSH_USER }}@${{ vars.SSH_HOST }}" \ No newline at end of file diff --git a/.github/workflows/test_compose.yml b/.github/workflows/test_compose.yml index 3b04f1dd50..d5b2fa8d4e 100644 --- a/.github/workflows/test_compose.yml +++ b/.github/workflows/test_compose.yml @@ -67,6 +67,11 @@ on: description: "Path to example dotenv file to substitute variables for." type: string default: .env.example + run_tests: + description: "If to run python tests." + required: false + type: boolean + default: false coverage: description: "Generate a coverage HTML report (requires coverage.py installed)." required: false @@ -273,10 +278,11 @@ jobs: # Run the tests via a pre-configured docker compose service # For --no-TTY explanation, see: https://github.com/actions/runner-images/issues/5022 - name: Run Tests - if: ${{ ! inputs.coverage && ! inputs.playwright }} + if: ${{ ! inputs.run_tests }} run: | ${{ inputs.pre_command }} + # Docker compose command to run tests. docker compose --file ${{ inputs.compose_file }} \ exec -i ${{ inputs.compose_service }} ${{ inputs.compose_command }}