Merge pull request #8 from IATI/develop

feat: move all served files to $web; add CDN, custom domain

.env-example

@@ -1,7 +1,7 @@
 DATA_REGISTRATION=ckan-registry
 DATA_REGISTRY_BASE_URL=https://iatiregistry.org/api/3/action/package_search
 
-BLOB_STORAGE_BASE_PUBLIC_URL=http://127.0.0.1:10000/devstoreaccount1
+WEB_BASE_URL=http://127.0.0.1:10000/devstoreaccount1
 
 NUMBER_DOWNLOADER_THREADS=1  # makes for easier testing locally
 

.gitignore

@@ -9,6 +9,8 @@ __pycache__
 
 /.env
 
+/azure-provision/default-github-config.env
+
 /azure-deployment/azure-resource-manager-deployment-manifest.yml
 /azure-deployment/manual-azure-deploy-secrets.env
 /azure-deployment/manual-azure-deploy-variables.env

README.md

@@ -181,6 +181,8 @@ This automated test environment is configured via the following files:
 
 You can use the Mockoon GUI application to edit the mockoon server configuration file (`mockoon-registration-and-data-server-config.json`).
 
+The automated tests are safe to run alongside the `docker compose` setup for development.
+
 ## Provisioning and Deployment
 
 ### Initial Provisioning

azure-deployment/azure-resource-manager-deployment-template.yml

@@ -32,8 +32,8 @@ properties: # Properties of container group
             value: "#DATA_REGISTRATION#"
           - name: DATA_REGISTRY_BASE_URL
             value: "#DATA_REGISTRY_BASE_URL#"
-          - name: BLOB_STORAGE_BASE_PUBLIC_URL
-            value: "https://sabulkdataservice#TARGET_ENVIRONMENT#.blob.core.windows.net"
+          - name: WEB_BASE_URL
+            value: "#WEB_BASE_URL#"
           - name: NUMBER_DOWNLOADER_THREADS
             value: "#NUMBER_DOWNLOADER_THREADS#"
           - name: FORCE_REDOWNLOAD_AFTER_HOURS

azure-deployment/generate-manifest-from-template.sh

@@ -49,3 +49,4 @@ sed -i ''s^#REMOVE_LAST_GOOD_DOWNLOAD_AFTER_FAILING_HOURS#^$REMOVE_LAST_GOOD_DOW
 sed -i ''s^#ZIP_WORKING_DIR#^$ZIP_WORKING_DIR^g'' ./azure-deployment/azure-resource-manager-deployment-manifest.yml
 sed -i ''s^#AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML#^$AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML^g'' ./azure-deployment/azure-resource-manager-deployment-manifest.yml
 sed -i ''s^#AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP#^$AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP^g'' ./azure-deployment/azure-resource-manager-deployment-manifest.yml
+sed -i ''s^#WEB_BASE_URL#^$WEB_BASE_URL^g'' ./azure-deployment/azure-resource-manager-deployment-manifest.yml

azure-deployment/manual-azure-deploy-from-local.sh

@@ -13,44 +13,54 @@ if [ ! -d ".git" ]; then
     exit 1
 fi
 
-git remote -v | grep "IATI/bulk-data-service.git" > /dev/null
+if [ ! -f "./azure-deployment/manual-azure-deploy-secrets.env" ]; then
+    echo "$0: there must be a file 'manual-azure-deploy-secrets.env' in"
+    echo "'azure-deployment' containing the secrets. See the examples in manual-azure-deploy-secrets-example.env'"
+    exit 1
+fi
 
-if [ "$?" != 0 ]; then
-    echo "$0: script must be run from the root of the bulk-data-service repository"
+if [ ! -f "./azure-deployment/manual-azure-deploy-variables.env" ]; then
+    echo "$0: there must be a file 'manual-azure-deploy-variables.env' in"
+    echo "'azure-deployment' containing the config variables. See example: manual-azure-deploy-variables-example.env'"
     exit 1
 fi
 
-. ./manual-azure-deploy-secrets.env
+
+(git remote -v 2> /dev/null | grep "IATI/bulk-data-service.git" > /dev/null) || (echo "$0: script must be run from the root of the bulk-data-service repository"; exit 1)
+
+. ./azure-deployment/manual-azure-deploy-secrets.env
 
 TARGET_ENVIRONMENT=$1
 
 APP_NAME=bulk-data-service
 
-RESOURCE_GROUP_NAME=rg-${APP_NAME}-${TARGET_ENVIRONMENT}
+RESOURCE_GROUP_NAME="rg-${APP_NAME}-${TARGET_ENVIRONMENT}"
+
+CONTAINER_GROUP_INSTANCE_NAME="aci-${APP_NAME}-${TARGET_ENVIRONMENT}"
 
-CONTAINER_GROUP_INSTANCE_NAME=aci-${APP_NAME}-${TARGET_ENVIRONMENT}
+DOCKER_IMAGE_TAG=$(git log -n1 --format=format:"%H")
 
 LOCAL_DEPLOY=true
 
 echo "Generating Azure ARM deployment manifest from template"
 . ./azure-deployment/generate-manifest-from-template.sh
 
 # build the docker image for the Bulk Data Service
-docker build . -t criati.azurecr.io/bulk-data-service-$TARGET_ENVIRONMENT
+docker build . -t "criati.azurecr.io/bulk-data-service-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG"
 
 # push Bulk Data Service image to Azure
-docker push criati.azurecr.io/bulk-data-service-$TARGET_ENVIRONMENT
+docker push "criati.azurecr.io/bulk-data-service-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG"
 
 # now configure, build and push the docker image for the nginx reverse proxy
 
 # create password file
-htpasswd -c -b ./azure-deployment/nginx-reverse-proxy/htpasswd prom $PROM_NGINX_REVERSE_PROXY_PASSWORD
+htpasswd -c -b ./azure-deployment/nginx-reverse-proxy/htpasswd prom "$PROM_NGINX_REVERSE_PROXY_PASSWORD"
 
 # make the image for the nginx reverse proxy (for putting HTTP basic auth on the
 # prom client)
-docker build ./azure-deployment/nginx-reverse-proxy -t criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT
+docker build ./azure-deployment/nginx-reverse-proxy -t "criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG"
 
-docker push criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT
+docker push "criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG"
 
 
 echo az container delete \

azure-deployment/manual-azure-deploy-variables-example.env

@@ -4,8 +4,7 @@
 AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML=iati-xml
 AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP=iati-zip
 
-# Value of BLOB_STORAGE_BASE_PUBLIC_URL generated automatically by deploy scripts
-# BLOB_STORAGE_BASE_PUBLIC_URL=
+WEB_BASE_URL=
 
 DATA_REGISTRATION=ckan-registry
 DATA_REGISTRY_BASE_URL=https://iatiregistry.org/api/3/action/package_search

azure-provision/add-default-config-to-github-variables.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/bash
+
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
+
+if [[ ! -v "1" ]]; then
+     echo "usage: $0 TARGET_ENVIRONMENT"
+     echo "       TARGET_ENVIRONMENT should likely be 'test', 'dev', or 'prod'"
+     exit 1
+fi
+
+if [[ ! -d ".git" ]]; then
+    echo "$0: script must be run from the root of the bulk-data-service repository"
+    exit 1
+fi
+
+(git remote -v 2> /dev/null | grep "IATI/bulk-data-service.git" > /dev/null) || (echo "$0: script must be run from the root of the bulk-data-service repository"; exit 1)
+
+if [[ "$1" == "" ]]; then
+     echo "TARGET_ENVIRONMENT cannot be empty"
+     exit 2
+fi
+
+if [[ $(which gh > /dev/null) ]]; then
+     echo "This script requires the Github command line client to be installed"
+     exit 3
+fi
+
+TARGET_ENVIRONMENT="$1"
+
+cp -f azure-provision/default-github-config-template.env azure-provision/default-github-config.env
+
+sed -i "s/^/${TARGET_ENVIRONMENT^^}/g" azure-provision/default-github-config.env
+
+gh variable set --env-file ./azure-provision/default-github-config.env