Merge pull request #1 from IATI/initial_version

Initial version

.env-example

@@ -0,0 +1,29 @@
+DATA_REGISTRATION=ckan-registry
+DATA_REGISTRY_BASE_URL=https://iatiregistry.org/api/3/action/package_search
+
+BLOB_STORAGE_BASE_PUBLIC_URL=http://127.0.0.1:10000/devstoreaccount1
+
+NUMBER_DOWNLOADER_THREADS=1  # makes for easier testing locally
+
+FORCE_REDOWNLOAD_AFTER_HOURS=24
+
+REMOVE_LAST_GOOD_DOWNLOAD_AFTER_FAILING_HOURS=72
+
+# Log file
+LOGFILE=
+
+# Sample local setup - values read by docker compose (for simple Postgres DB
+# creation), and used by the app
+DB_NAME=bulk_data_service_db
+DB_USER=bds
+DB_PASS=pass
+DB_HOST=localhost
+DB_PORT=5255
+DB_SSL_MODE=disable
+DB_CONNECTION_TIMEOUT=30
+
+# Local Azurite Emulator
+AZURE_STORAGE_CONNECTION_STRING=AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;
+
+AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML=iati-xml
+AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP=iati-zip

.github/workflows/build-and-deploy-job.yml

@@ -0,0 +1,123 @@
+name: Generic build and deploy (called by other workflows)
+
+on:
+  workflow_call:
+    inputs:
+      APP_NAME:
+        required: true
+        type: string
+      TARGET_ENVIRONMENT:
+        required: true
+        type: string
+
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    env:
+      APP_NAME: ${{ inputs.APP_NAME }}
+      TARGET_ENVIRONMENT: ${{ inputs.TARGET_ENVIRONMENT }}
+
+      DOCKER_IMAGE_TAG: ${{ github.sha }}
+
+      # Needed as an environment variable for use of 'az' cmd in inline shell script
+      ACR_LOGIN_SERVER: ${{ secrets.ACR_LOGIN_SERVER }}
+      ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
+      ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
+
+    steps:
+      - name: 'Generate/build derived environment variables'
+        run: |
+          echo "TARGET_ENVIRONMENT_UPPER=${TARGET_ENVIRONMENT^^}" >> ${GITHUB_ENV}
+          echo "CONTAINER_INSTANCE_BASE_NAME=aci-${APP_NAME}" >> ${GITHUB_ENV}
+          echo "RESOURCE_GROUP_BASE_NAME=rg-${APP_NAME}" >> ${GITHUB_ENV}
+
+      - name: 'Print calculated environment variables'
+        run: |
+          echo $TARGET_ENVIRONMENT_UPPER
+          echo $CONTAINER_INSTANCE_BASE_NAME
+          echo $RESOURCE_GROUP_BASE_NAME
+
+      - name: 'Checkout GitHub Action'
+        uses: actions/checkout@v4
+
+      - name: 'Login via Azure CLI'
+        uses: azure/login@v2
+        with:
+          creds: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_CREDENTIALS')] }}
+
+      - name: 'Login to Docker Hub'
+        uses: docker/[email protected]
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: 'Login to Azure Container Registry'
+        uses: azure/docker-login@v2
+        with:
+          login-server: ${{ env.ACR_LOGIN_SERVER }}
+          username: ${{ env.ACR_USERNAME }}
+          password: ${{ env.ACR_PASSWORD }}
+
+      - name: 'Build and push image'
+        run: |
+          IMAGE_NAME=$ACR_LOGIN_SERVER/$APP_NAME-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG
+          echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV
+          docker build . -f Dockerfile -t $IMAGE_NAME
+          docker push $IMAGE_NAME
+
+      - name: 'Print IMAGE_NAME for Bulk Data Service image'
+        run: echo $IMAGE_NAME
+
+      - name: 'Create htpasswd file for nginx reverse proxy'
+        run: |
+          htpasswd -c -b ./azure-deployment/nginx-reverse-proxy/htpasswd prom "${{ secrets.PROM_NGINX_REVERSE_PROXY_PASSWORD }}"
+          docker build ./azure-deployment/nginx-reverse-proxy -t criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG
+          docker push criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG
+
+      - name: 'Delete existing container group'
+        uses: 'azure/CLI@v2'
+        with:
+          inlineScript: |
+            az -v
+            az container delete -y \
+              --name "${{ env.CONTAINER_INSTANCE_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" \
+              --resource-group "${{ env.RESOURCE_GROUP_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}"
+
+      - name: 'Replace Env Vars and Secrets in ARM Yaml template'
+        env:
+          # Credentials for the app's resources
+          AZURE_STORAGE_CONNECTION_STRING: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_STORAGE_CONNECTION_STRING')] }}
+
+          DB_HOST: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_HOST')] }}
+          DB_USER: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_USER')] }}
+          DB_PASS: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_PASS')] }}
+          DB_NAME: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_NAME')] }}
+          DB_PORT: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_PORT')] }}
+          DB_SSL_MODE: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_SSL_MODE')] }}
+          DB_CONNECTION_TIMEOUT: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DB_CONNECTION_TIMEOUT')] }}
+
+          LOG_WORKSPACE_ID: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'LOG_WORKSPACE_ID')] }}
+          LOG_WORKSPACE_KEY: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'LOG_WORKSPACE_KEY')] }}
+
+          # Variables which configure the app
+          DATA_REGISTRATION: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DATA_REGISTRATION')] }}
+          DATA_REGISTRY_BASE_URL: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'DATA_REGISTRY_BASE_URL')] }}
+          NUMBER_DOWNLOADER_THREADS: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'NUMBER_DOWNLOADER_THREADS')] }}
+          FORCE_REDOWNLOAD_AFTER_HOURS: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'FORCE_REDOWNLOAD_AFTER_HOURS')] }}
+          REMOVE_LAST_GOOD_DOWNLOAD_AFTER_FAILING_HOURS: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'REMOVE_LAST_GOOD_DOWNLOAD_AFTER_FAILING_HOURS')] }}
+          ZIP_WORKING_DIR: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'ZIP_WORKING_DIR')] }}
+          AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML')] }}
+          AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP')] }}
+
+        run: |
+          ./azure-deployment/generate-manifest-from-template.sh
+
+      - name: 'Deploy group to Azure Container Instances'
+        uses: 'azure/CLI@v2'
+        with:
+          inlineScript: |
+            az -v
+            az container create --debug \
+            --resource-group "${{ env.RESOURCE_GROUP_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" \
+            --file ./azure-deployment/azure-resource-manager-deployment-manifest.yml

.github/workflows/deploy-to-dev.yml

@@ -0,0 +1,22 @@
+name: Deploy Bulk Data Service to dev
+
+
+on:
+  workflow_dispatch:
+  push:
+    paths-ignore:
+      - '.github/workflows/deploy-to-prod.yml'
+    branches:
+      - develop
+
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/test.yml
+  call-build-and-deploy:
+    needs: run-tests
+    uses: ./.github/workflows/build-and-deploy-job.yml
+    secrets: inherit
+    with:
+      APP_NAME: "bulk-data-service"
+      TARGET_ENVIRONMENT: "test"

.github/workflows/deploy-to-prod.yml

@@ -0,0 +1,16 @@
+name: Deploy Bulk Data Service to production
+
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+
+jobs:
+  call-build-and-deploy:
+    uses: ./.github/workflows/build-and-deploy-job.yml
+    secrets: inherit
+    with:
+      APP_NAME: "bulk-data-service"
+      TARGET_ENVIRONMENT: "test"

.github/workflows/test.yml

@@ -0,0 +1,31 @@
+name: Run Automated Tests
+
+on:
+  workflow_call:
+  workflow_dispatch:
+  push:
+    branches:
+      - '**'
+      - '!develop'
+      - '!main'
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Setup python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+        architecture: x64
+
+    - name: Install requirements-dev.txt
+      run: pip install -r requirements-dev.txt
+
+    - name: Run docker-compose
+      run: cd ./tests-local-environment; docker compose up -d
+
+    - name: Run automated tests
+      run: pytest

.gitignore

@@ -0,0 +1,17 @@
+__pycache__
+*.py[cod]
+
+/.actrc
+
+.mypy_cache
+.pytest_cache
+.ve
+
+/.env
+
+/azure-deployment/azure-resource-manager-deployment-manifest.yml
+/azure-deployment/manual-azure-deploy-secrets.env
+/azure-deployment/manual-azure-deploy-variables.env
+/azure-deployment/nginx-reverse-proxy/htpasswd
+
+/web/index.html

.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Bulk Data Service - Checker - Single Run",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "src/iati_bulk_data_service.py",
+            "args": [
+                "--operation",
+                "checker",
+                "--single-run"
+            ],
+            "console": "integratedTerminal",
+            "envFile": "${workspaceFolder}/.env"
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,31 @@
+{
+    "python.testing.pytestArgs": [
+        "tests"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "files.trimTrailingWhitespace": true,
+    "python.analysis.typeCheckingMode": "basic",
+    "editor.formatOnSave": true,
+    "[python]": {
+        "editor.formatOnSave": true
+    },
+    "editor.codeActionsOnSave": {
+        "source.organizeImports": "always"
+    },
+    "isort.args": [
+        "--profile",
+        "black",
+        "--py",
+        "312"
+    ],
+    "isort.path": [
+        "isort"
+    ],
+    "isort.interpreter": [
+        "python"
+    ],
+    "mypy.targets": [
+        "src/"
+    ],
+}

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.12-slim-bookworm
+
+RUN apt-get update -y
+
+WORKDIR /bulk-data-service
+
+COPY requirements.txt .
+
+RUN pip install -r requirements.txt
+
+COPY src/ src
+COPY db-migrations/ db-migrations
+
+ENTRYPOINT ["/usr/local/bin/python", "src/iati_bulk_data_service.py"]