-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: provisioning, deployment on GH actions
* Improvement to the provisioning script * Added example environment and configuration files * Added Github Actions deployment workflows for dev, prod
- Loading branch information
Showing
12 changed files
with
453 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| name: Generic build and deploy (called by other workflows) | ||
|
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| APP_NAME: | ||
| required: true | ||
| type: string | ||
| TARGET_ENVIRONMENT: | ||
| required: true | ||
| type: string | ||
|
|
||
|
|
||
| jobs: | ||
| build-and-deploy: | ||
| runs-on: ubuntu-latest | ||
|
|
||
| env: | ||
| APP_NAME: ${{ inputs.APP_NAME }} | ||
| TARGET_ENVIRONMENT: ${{ inputs.TARGET_ENVIRONMENT }} | ||
|
|
||
| DOCKER_IMAGE_TAG: ${{ github.sha }} | ||
|
|
||
| # Needed as an environment variable for use of 'az' cmd in inline shell script | ||
| ACR_LOGIN_SERVER: ${{ secrets.ACR_LOGIN_SERVER }} | ||
| ACR_USERNAME: ${{ secrets.ACR_USERNAME }} | ||
| ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }} | ||
|
|
||
| steps: | ||
| - name: 'Generate/build derived environment variables' | ||
| run: | | ||
| echo "TARGET_ENVIRONMENT_UPPER=${TARGET_ENVIRONMENT^^}" >> ${GITHUB_ENV} | ||
| echo "CONTAINER_INSTANCE_BASE_NAME=aci-${APP_NAME}" >> ${GITHUB_ENV} | ||
| echo "RESOURCE_GROUP_BASE_NAME=rg-${APP_NAME}" >> ${GITHUB_ENV} | ||
| - name: 'Print calculated environment variables' | ||
| run: | | ||
| echo $TARGET_ENVIRONMENT_UPPER | ||
| echo $CONTAINER_INSTANCE_BASE_NAME | ||
| echo $RESOURCE_GROUP_BASE_NAME | ||
| - name: 'Checkout GitHub Action' | ||
| uses: actions/checkout@v4 | ||
|
|
||
| - name: 'Login via Azure CLI' | ||
| uses: azure/login@v2 | ||
| with: | ||
| creds: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_CREDENTIALS')] }} | ||
|
|
||
| - name: 'Login to Docker Hub' | ||
| uses: docker/[email protected] | ||
| with: | ||
| username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
| password: ${{ secrets.DOCKER_HUB_TOKEN }} | ||
|
|
||
| - name: 'Login to Azure Container Registry' | ||
| uses: azure/docker-login@v2 | ||
| with: | ||
| login-server: ${{ env.ACR_LOGIN_SERVER }} | ||
| username: ${{ env.ACR_USERNAME }} | ||
| password: ${{ env.ACR_PASSWORD }} | ||
|
|
||
| - name: 'Build and push image' | ||
| run: | | ||
| IMAGE_NAME=$ACR_LOGIN_SERVER/$APP_NAME-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG | ||
| echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV | ||
| docker build . -f Dockerfile -t $IMAGE_NAME | ||
| docker push $IMAGE_NAME | ||
| - name: 'Print IMAGE_NAME' | ||
| run: echo $IMAGE_NAME | ||
|
|
||
| - name: 'Delete existing container group' | ||
| uses: 'azure/CLI@v2' | ||
| with: | ||
| inlineScript: | | ||
| az -v | ||
| az container delete -y \ | ||
| --name "${{ env.CONTAINER_INSTANCE_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" \ | ||
| --resource-group "${{ env.RESOURCE_GROUP_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" | ||
| - name: 'Replace Env Vars and Secrets in ARM Yaml template' | ||
| env: | ||
| # Credentials for the app's resources | ||
| AZURE_STORAGE_CONNECTION_STRING: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_STORAGE_CONNECTION_STRING')] }} | ||
|
|
||
| DB_HOST: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_HOST')] }} | ||
| DB_USER: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_USER')] }} | ||
| DB_PASS: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_PASS')] }} | ||
| DB_NAME: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_NAME')] }} | ||
| DB_PORT: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_PORT')] }} | ||
| DB_SSL_MODE: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_SSL_MODE')] }} | ||
| DB_CONNECTION_TIMEOUT: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_CONNECTION_TIMEOUT')] }} | ||
|
|
||
| LOG_WORKSPACE_ID: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'LOG_WORKSPACE_ID')] }} | ||
| LOG_WORKSPACE_KEY: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'LOG_WORKSPACE_KEY')] }} | ||
|
|
||
| # Variables which configure the app | ||
| DATA_REGISTRATION: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DATA_REGISTRATION')] }} | ||
| DATA_REGISTRY_BASE_URL: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DATA_REGISTRY_BASE_URL')] }} | ||
| NUMBER_DOWNLOADER_THREADS: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'NUMBER_DOWNLOADER_THREADS')] }} | ||
| FORCE_REDOWNLOAD_AFTER_HOURS: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'FORCE_REDOWNLOAD_AFTER_HOURS')] }} | ||
| REMOVE_LAST_GOOD_DOWNLOAD_AFTER_FAILING_HOURS: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'REMOVE_LAST_GOOD_DOWNLOAD_AFTER_FAILING_HOURS')] }} | ||
| ZIP_WORKING_DIR: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'ZIP_WORKING_DIR')] }} | ||
| AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML')] }} | ||
| AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP')] }} | ||
|
|
||
| run: | | ||
| ./azure-deployment/generate-manifest-from-template.sh | ||
| - name: 'Deploy group to Azure Container Instances' | ||
| uses: 'azure/CLI@v2' | ||
| with: | ||
| inlineScript: | | ||
| az -v | ||
| az container create --debug \ | ||
| --resource-group "${{ env.RESOURCE_GROUP_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" \ | ||
| --file ./azure-deployment/azure-resource-manager-deployment-manifest.yml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| name: Deploy Bulk Data Service to dev | ||
|
|
||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| push: | ||
| paths-ignore: | ||
| - '.github/workflows/deploy-to-prod.yml' | ||
| branches: | ||
| - develop | ||
|
|
||
|
|
||
| jobs: | ||
| call-build-and-deploy: | ||
| uses: ./.github/workflows/build-and-deploy-job.yml | ||
| secrets: inherit | ||
| with: | ||
| APP_NAME: "bulk-data-service" | ||
| TARGET_ENVIRONMENT: "test" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| name: Deploy Bulk Data Service to production | ||
|
|
||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| release: | ||
| types: [published] | ||
|
|
||
|
|
||
| jobs: | ||
| call-build-and-deploy: | ||
| uses: ./.github/workflows/build-and-deploy-job.yml | ||
| secrets: inherit | ||
| with: | ||
| APP_NAME: "bulk-data-service" | ||
| TARGET_ENVIRONMENT: "test" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| name: Run Automated Tests | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| push: | ||
| branches: | ||
| develop | ||
|
|
||
| jobs: | ||
| run-tests: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
|
|
||
| - name: Setup python | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: 3.12 | ||
| architecture: x64 | ||
|
|
||
| - name: Install requirements-dev.txt | ||
| run: pip install -r requirements-dev.txt | ||
|
|
||
| - name: Run docker-compose | ||
| run: cd ./tests-local-environment; docker compose up -d | ||
|
|
||
| - name: Run automated tests | ||
| run: pytest |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
73 changes: 73 additions & 0 deletions
73
azure-deployment/azure-resource-manager-deployment-template.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| name: "aci-#APP_NAME#-#TARGET_ENVIRONMENT#" | ||
| apiVersion: "2021-10-01" | ||
| location: "uksouth" | ||
| properties: # Properties of container group | ||
| imageRegistryCredentials: # Credentials to pull a private image | ||
| - server: "#ACR_LOGIN_SERVER#" | ||
| username: "#ACR_USERNAME#" | ||
| password: "#ACR_PASSWORD#" | ||
| restartPolicy: "Never" | ||
| osType: "Linux" | ||
| diagnostics: | ||
| logAnalytics: | ||
| workspaceId: "#LOG_WORKSPACE_ID#" | ||
| workspaceKey: "#LOG_WORKSPACE_KEY#" | ||
| containers: | ||
| - name: "#APP_NAME#-#TARGET_ENVIRONMENT#" | ||
| properties: # Properties of an instance | ||
| resources: # Resource requirements of the instance | ||
| requests: | ||
| memoryInGB: 4 | ||
| cpu: 1 | ||
| image: "#ACR_LOGIN_SERVER#/#APP_NAME#-#TARGET_ENVIRONMENT#" | ||
| ports: | ||
| - port: 9090 | ||
| command: | ||
| - "/usr/local/bin/python" | ||
| - "src/iati_bulk_data_service.py" | ||
| - "--operation" | ||
| - "checker" | ||
| environmentVariables: | ||
| - name: DATA_REGISTRATION | ||
| value: "#DATA_REGISTRATION#" | ||
| - name: DATA_REGISTRY_BASE_URL | ||
| value: "#DATA_REGISTRY_BASE_URL#" | ||
| - name: BLOB_STORAGE_BASE_PUBLIC_URL | ||
| value: "https://sabulkdataservice#TARGET_ENVIRONMENT#.blob.core.windows.net" | ||
| - name: NUMBER_DOWNLOADER_THREADS | ||
| value: "#NUMBER_DOWNLOADER_THREADS#" | ||
| - name: FORCE_REDOWNLOAD_AFTER_HOURS | ||
| value: "#FORCE_REDOWNLOAD_AFTER_HOURS#" | ||
| - name: REMOVE_LAST_GOOD_DOWNLOAD_AFTER_FAILING_HOURS | ||
| value: "#REMOVE_LAST_GOOD_DOWNLOAD_AFTER_FAILING_HOURS#" | ||
| - name: LOGFILE | ||
| value: "" | ||
| - name: ZIP_WORKING_DIR | ||
| value: "#ZIP_WORKING_DIR#" | ||
| - name: AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML | ||
| value: "#AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML#" | ||
| - name: AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP | ||
| value: "#AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP#" | ||
|
|
||
| - name: AZURE_STORAGE_CONNECTION_STRING | ||
| secureValue: "#AZURE_STORAGE_CONNECTION_STRING#" | ||
| - name: DB_HOST | ||
| secureValue: "#APP_NAME#-db-#TARGET_ENVIRONMENT#.postgres.database.azure.com" | ||
| - name: DB_PORT | ||
| secureValue: "#DB_PORT#" | ||
| - name: DB_USER | ||
| secureValue: "#DB_USER#" | ||
| - name: DB_PASS | ||
| secureValue: "#DB_PASS#" | ||
| - name: DB_NAME | ||
| secureValue: "#DB_NAME#" | ||
| - name: DB_SSL_MODE | ||
| secureValue: "#DB_SSL_MODE#" | ||
| - name: DB_CONNECTION_TIMEOUT | ||
| secureValue: "#DB_CONNECTION_TIMEOUT#" | ||
|
|
||
| ipAddress: | ||
| type: "public" | ||
| dnsNameLabel: "#APP_NAME#-#TARGET_ENVIRONMENT#" | ||
| ports: | ||
| - port: 9090 |
Oops, something went wrong.