Release-v1.4.0

.github/workflows/provision.yml

@@ -74,7 +74,6 @@ jobs:
           mongodb_admin_username: ${{ secrets.MONGODB_ADMIN_USER }}
           mongodb_admin_password: ${{ secrets.MONGODB_ADMIN_PASSWORD }}
           backup_encryption_passphrase: ${{ secrets.BACKUP_ENCRYPTION_PASSPHRASE }}
-          restore_backup_encryption_passphrase: ${{ secrets.RESTORE_BACKUP_ENCRYPTION_PASSPHRASE }}
           elasticsearch_superuser_password: ${{ secrets.ELASTICSEARCH_SUPERUSER_PASSWORD }}
           external_backup_server_ssh_port: 22
           external_backup_server_ip: ${{ secrets.BACKUP_HOST }}

infrastructure/deployment/deploy.sh

@@ -352,6 +352,12 @@ EOF
 
 # Setup configuration files and compose file for the deployment domain
 configured_ssh "
+  HOST=$HOST
+  SMTP_HOST=$SMTP_HOST
+  SMTP_PORT=$SMTP_PORT
+  ALERT_EMAIL=$ALERT_EMAIL
+  SENDER_EMAIL_ADDRESS=$SENDER_EMAIL_ADDRESS
+  DOMAIN=$DOMAIN
   MINIO_ROOT_USER=$MINIO_ROOT_USER
   MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD
   /opt/opencrvs/infrastructure/setup-deploy-config.sh $HOST | tee -a $LOG_LOCATION/setup-deploy-config.log"
@@ -366,6 +372,7 @@ echo
 echo "Waiting 2 mins for mongo to deploy before working with data. Please note it can take up to 10 minutes for the entire stack to deploy in some scenarios."
 echo
 
+sleep 120 # Required as Kibana cannot be immediately contacted
 echo "Setting up Kibana config & alerts"
 
 while true; do

infrastructure/docker-compose.deploy.yml

@@ -405,6 +405,8 @@ services:
       - '/opt/opencrvs/infrastructure/monitoring/elastalert/rules:/opt/elastalert/rules'
     networks:
       - overlay_net
+    depends_on:
+      - elasticsearch
     deploy:
       labels:
         - 'traefik.enable=false'

infrastructure/docker-compose.development-deploy.yml

@@ -15,6 +15,7 @@ services:
       - jwt-public-key.{{ts}}
     environment:
       - NODE_ENV=production
+      - QA_ENV=true
       - FHIR_URL=http://hearth:3447/fhir
       - AUTH_URL=http://auth:4040
       - APPLICATION_CONFIG_URL=http://config:2021

infrastructure/monitoring/elastalert/elastalert.yaml

@@ -16,8 +16,6 @@ buffer_time:
 
 es_host: elasticsearch
 es_port: 9200
-es_username: '{{ES_USERNAME}}'
-es_password: '{{ES_PASSWORD}}'
 
 writeback_index: elastalert_status
 

infrastructure/monitoring/kibana/setup-config.sh

@@ -14,14 +14,14 @@ set -e
 docker_command="docker run --rm -v /opt/opencrvs/infrastructure/monitoring/kibana/config.ndjson:/config.ndjson --network=opencrvs_overlay_net curlimages/curl"
 
 # First delete all alerts. This is because the import doesn't remove alerts that are no longer in the config
-$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do
-  $docker_command --connect-timeout 60 -X DELETE -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id"
+$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do
+  $docker_command --connect-timeout 60 -X DELETE -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id"
 done
 
-$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true -H 'kbn-xsrf: true' --form file=@/config.ndjson > /dev/null
+$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true -H 'kbn-xsrf: true' --form file=@/config.ndjson > /dev/null
 
 # Re-enable all alerts. This is because after importing a config, all alerts are disabled by default
-$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do
-  $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id/_disable"
-  $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id/_enable"
+$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do
+  $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id/_disable"
+  $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id/_enable"
 done

infrastructure/server-setup/backups.yml

@@ -87,16 +87,6 @@
   vars:
     manager_hostname: "{{ groups['docker-manager-first'][0] }}"
   tasks:
-    - name: Ensure backup user is present
-      user:
-        name: '{{ external_backup_server_user }}'
-        state: present
-        create_home: true
-        home: '/home/{{ external_backup_server_user }}'
-        shell: /bin/bash
-      tags:
-        - backups
-
     - set_fact:
         external_backup_server_user_home: '/home/{{ external_backup_server_user }}'
       tags:

infrastructure/server-setup/playbook.yml

@@ -34,7 +34,9 @@
 
 - hosts: docker-manager-first, docker-workers
   become: yes
-  become_method: sudo
+  become_method: sudo  
+  vars:
+    crontab_user: root
   tasks:
     - include_tasks:
         file: tasks/application.yml

infrastructure/server-setup/staging.yml

@@ -15,7 +15,6 @@ all:
       - 165.22.110.53
     enable_backups: false
     periodic_restore_from_backup: true
-    # restore_backup_encryption_passphrase: Defined in --extra-vars by the provisioning pipeline
     # external_backup_server_ssh_port: Defined in --extra-vars by the provisioning pipeline
     # external_backup_server_ip: Defined in --extra-vars by the provisioning pipeline
     users:

infrastructure/server-setup/tasks/backups/crontab.yml

@@ -16,19 +16,19 @@
     periodic_restore_from_backup: false
   when: periodic_restore_from_backup is not defined
 
-- name: Throw an error if periodic_restore_from_backup is true but restore_backup_encryption_passphrase is not defined
+- name: Throw an error if periodic_restore_from_backup is true but backup_encryption_passphrase is not defined
   fail:
-    msg: 'Error: restore_backup_encryption_passphrase is not defined. It usually means you haven't set RESTORE_BACKUP_ENCRYPTION_PASSPHRASE in your staging environments secrets'
-  when: periodic_restore_from_backup and restore_backup_encryption_passphrase is not defined
+    msg: 'Error: backup_encryption_passphrase is not defined. It usually means you have not set backup_encryption_passphrase in your staging environments secrets'
+  when: periodic_restore_from_backup and backup_encryption_passphrase is not defined
 
 - name: 'Setup crontab to download a backup periodically the opencrvs data'
   cron:
     user: '{{ crontab_user }}'
     name: 'download opencrvs backup'
     minute: '30'
     hour: '0'
-    job: 'cd / && bash /opt/opencrvs/infrastructure/backups/download.sh --passphrase={{ restore_backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --remote_dir={{ external_backup_server_remote_directory }} >> /var/log/opencrvs-restore.log 2>&1'
-    state: "{{ 'present' if (external_backup_server_ip is defined and restore_backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"
+    job: 'cd / && bash /opt/opencrvs/infrastructure/backups/download.sh --passphrase={{ backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --remote_dir={{ external_backup_server_remote_directory }} >> /var/log/opencrvs-restore.log 2>&1'
+    state: "{{ 'present' if (external_backup_server_ip is defined and backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"
 
 - name: 'Setup crontab to restore the opencrvs data'
   cron:
@@ -37,4 +37,4 @@
     minute: '0'
     hour: '1'
     job: 'cd / && bash /opt/opencrvs/infrastructure/backups/restore.sh --replicas=1 >> /var/log/opencrvs-restore.log 2>&1'
-    state: "{{ 'present' if (external_backup_server_ip is defined and restore_backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"
+    state: "{{ 'present' if (external_backup_server_ip is defined and backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"

infrastructure/server-setup/tasks/updates.yml

@@ -25,6 +25,9 @@
     content: |
       Unattended-Upgrade::Package-Blacklist {};
       Unattended-Upgrade::DevRelease "auto";
+      Unattended-Upgrade::Remove-Unused-Kernel-Packages "true";
+      Unattended-Upgrade::Remove-New-Unused-Dependencies "true";
+      Unattended-Upgrade::Remove-Unused-Dependencies "true";
       Unattended-Upgrade::Allowed-Origins {
           "${distro_id}:${distro_codename}-security";
       };

infrastructure/setup-deploy-config.sh

@@ -26,6 +26,19 @@ done
 KIBANA_ENCRYPTION_KEY=`uuidgen`
 sed -i "s/{{KIBANA_ENCRYPTION_KEY}}/$KIBANA_ENCRYPTION_KEY/g" /opt/opencrvs/infrastructure/monitoring/kibana/kibana.yml
 
+# Move metabase file
+mv /opt/opencrvs/infrastructure/metabase.init.db.sql /data/metabase/metabase.init.db.sql
+
+# Replace environment variables from all alert definition files
+for file in /opt/opencrvs/infrastructure/monitoring/elastalert/rules/*.yaml; do
+    sed -i -e "s%{{HOST}}%$1%" $file
+    sed -i -e "s%{{SMTP_HOST}}%$SMTP_HOST%" $file
+    sed -i -e "s%{{SMTP_PORT}}%$SMTP_PORT%" $file
+    sed -i -e "s%{{ALERT_EMAIL}}%$ALERT_EMAIL%" $file
+    sed -i -e "s%{{SENDER_EMAIL_ADDRESS}}%$SENDER_EMAIL_ADDRESS%" $file
+    sed -i -e "s%{{DOMAIN}}%$DOMAIN%" $file
+done
+
 sed -i -e "s%{{MINIO_ROOT_USER}}%$MINIO_ROOT_USER%" /opt/opencrvs/infrastructure/mc-config/config.json
 sed -i -e "s%{{MINIO_ROOT_PASSWORD}}%$MINIO_ROOT_PASSWORD%" /opt/opencrvs/infrastructure/mc-config/config.json