Merge pull request #73 from opencrvs/fixes-to-release

Release-v1.4.0 updates

.github/workflows/deploy-prod.yml

@@ -30,8 +30,8 @@ jobs:
           secret: ${{ github.TOKEN }}
           approvers: euanmillar,rikukissa
           minimum-approvals: 1
-          issue-title: 'Deploy (Prod): core: ${{ github.event.inputs.core-image-tag }} country config: ${{ github.event.inputs.countryconfig-image-tag }}'
-          issue-body: 'Please approve or deny the deployment of core: ${{ github.event.inputs.core-image-tag }} country config: ${{ github.event.inputs.countryconfig-image-tag }} to production'
+          issue-title: 'Deploy (${{ github.event.inputs.environment }}): core: ${{ github.event.inputs.core-image-tag }} country config: ${{ github.event.inputs.countryconfig-image-tag }}'
+          issue-body: 'Please approve or deny the deployment of core: ${{ github.event.inputs.core-image-tag }} country config: ${{ github.event.inputs.countryconfig-image-tag }} to ${{ github.event.inputs.environment }}'
           exclude-workflow-initiator-as-approver: false
       - name: Clone core
         uses: actions/checkout@v3

.github/workflows/provision.yml

@@ -74,7 +74,6 @@ jobs:
           mongodb_admin_username: ${{ secrets.MONGODB_ADMIN_USER }}
           mongodb_admin_password: ${{ secrets.MONGODB_ADMIN_PASSWORD }}
           backup_encryption_passphrase: ${{ secrets.BACKUP_ENCRYPTION_PASSPHRASE }}
-          restore_backup_encryption_passphrase: ${{ secrets.RESTORE_BACKUP_ENCRYPTION_PASSPHRASE }}
           elasticsearch_superuser_password: ${{ secrets.ELASTICSEARCH_SUPERUSER_PASSWORD }}
           external_backup_server_ssh_port: 22
           external_backup_server_ip: ${{ secrets.BACKUP_HOST }}

infrastructure/deployment/deploy.sh

@@ -352,6 +352,12 @@ EOF
 
 # Setup configuration files and compose file for the deployment domain
 configured_ssh "
+  HOST=$HOST
+  SMTP_HOST=$SMTP_HOST
+  SMTP_PORT=$SMTP_PORT
+  ALERT_EMAIL=$ALERT_EMAIL
+  SENDER_EMAIL_ADDRESS=$SENDER_EMAIL_ADDRESS
+  DOMAIN=$DOMAIN
   MINIO_ROOT_USER=$MINIO_ROOT_USER
   MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD
   /opt/opencrvs/infrastructure/setup-deploy-config.sh $HOST | tee -a $LOG_LOCATION/setup-deploy-config.log"
@@ -366,6 +372,7 @@ echo
 echo "Waiting 2 mins for mongo to deploy before working with data. Please note it can take up to 10 minutes for the entire stack to deploy in some scenarios."
 echo
 
+sleep 120 # Required as Kibana cannot be immediately contacted
 echo "Setting up Kibana config & alerts"
 
 while true; do

infrastructure/docker-compose.deploy.yml

@@ -405,6 +405,8 @@ services:
       - '/opt/opencrvs/infrastructure/monitoring/elastalert/rules:/opt/elastalert/rules'
     networks:
       - overlay_net
+    depends_on:
+      - elasticsearch
     deploy:
       labels:
         - 'traefik.enable=false'

infrastructure/docker-compose.development-deploy.yml

@@ -15,6 +15,7 @@ services:
       - jwt-public-key.{{ts}}
     environment:
       - NODE_ENV=production
+      - QA_ENV=true
       - FHIR_URL=http://hearth:3447/fhir
       - AUTH_URL=http://auth:4040
       - APPLICATION_CONFIG_URL=http://config:2021

infrastructure/docker-compose.production-deploy.yml

@@ -39,7 +39,6 @@ services:
 
   metrics:
     environment:
-      - QA_ENV=true
       - NODE_ENV=production
       - SENTRY_DSN=${SENTRY_DSN}
       - MONGO_URL=mongodb://metrics:${METRICS_MONGODB_PASSWORD}@mongo1,mongo2/metrics?replicaSet=rs0
@@ -48,15 +47,13 @@ services:
 
   auth:
     environment:
-      - QA_ENV=true
       - NODE_ENV=production
       - SENTRY_DSN=${SENTRY_DSN}
     deploy:
       replicas: 2
 
   user-mgnt:
     environment:
-      - QA_ENV=true
       - NODE_ENV=production
       - SENTRY_DSN=${SENTRY_DSN}
       - MONGO_URL=mongodb://user-mgnt:${USER_MGNT_MONGODB_PASSWORD}@mongo1,mongo2/user-mgnt?replicaSet=rs0

infrastructure/docker-compose.staging-deploy.yml

@@ -39,7 +39,6 @@ services:
 
   metrics:
     environment:
-      - QA_ENV=true
       - NODE_ENV=production
       - SENTRY_DSN=${SENTRY_DSN}
       - MONGO_URL=mongodb://metrics:${METRICS_MONGODB_PASSWORD}@mongo1/metrics?replicaSet=rs0
@@ -48,15 +47,13 @@ services:
 
   auth:
     environment:
-      - QA_ENV=true
       - NODE_ENV=production
       - SENTRY_DSN=${SENTRY_DSN}
     deploy:
       replicas: 1
 
   user-mgnt:
     environment:
-      - QA_ENV=true
       - NODE_ENV=production
       - SENTRY_DSN=${SENTRY_DSN}
       - MONGO_URL=mongodb://user-mgnt:${USER_MGNT_MONGODB_PASSWORD}@mongo1/user-mgnt?replicaSet=rs0

infrastructure/monitoring/elastalert/elastalert.yaml

@@ -16,8 +16,6 @@ buffer_time:
 
 es_host: elasticsearch
 es_port: 9200
-es_username: '{{ES_USERNAME}}'
-es_password: '{{ES_PASSWORD}}'
 
 writeback_index: elastalert_status
 

infrastructure/monitoring/kibana/setup-config.sh

@@ -14,14 +14,14 @@ set -e
 docker_command="docker run --rm -v /opt/opencrvs/infrastructure/monitoring/kibana/config.ndjson:/config.ndjson --network=opencrvs_overlay_net curlimages/curl"
 
 # First delete all alerts. This is because the import doesn't remove alerts that are no longer in the config
-$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do
-  $docker_command --connect-timeout 60 -X DELETE -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id"
+$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do
+  $docker_command --connect-timeout 60 -X DELETE -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id"
 done
 
-$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true -H 'kbn-xsrf: true' --form file=@/config.ndjson > /dev/null
+$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true -H 'kbn-xsrf: true' --form file=@/config.ndjson > /dev/null
 
 # Re-enable all alerts. This is because after importing a config, all alerts are disabled by default
-$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do
-  $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id/_disable"
-  $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id/_enable"
+$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do
+  $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id/_disable"
+  $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id/_enable"
 done

infrastructure/server-setup/backups.yml

@@ -87,37 +87,11 @@
   vars:
     manager_hostname: "{{ groups['docker-manager-first'][0] }}"
   tasks:
-    - name: Ensure backup user is present
-      user:
-        name: '{{ external_backup_server_user }}'
-        state: present
-        create_home: true
-        home: '/home/{{ external_backup_server_user }}'
-        shell: /bin/bash
-      tags:
-        - backups
-
     - set_fact:
         external_backup_server_user_home: '/home/{{ external_backup_server_user }}'
       tags:
         - backups
 
-    - name: Check if authorized_keys already exists
-      stat:
-        path: '{{ external_backup_server_user_home }}/.ssh/authorized_keys'
-      register: file_check
-      tags:
-        - backups
-
-    - name: Ensure .ssh directory exists for external backup server user
-      file:
-        path: '{{ external_backup_server_user_home }}/.ssh'
-        state: directory
-        owner: '{{ external_backup_server_user }}'
-        mode: '0700'
-      tags:
-        - backups
-
     - name: Ensure backup application servers can login to backup server
       blockinfile:
         path: '{{ external_backup_server_user_home }}/.ssh/authorized_keys'

infrastructure/server-setup/development.yml

@@ -18,8 +18,8 @@ all:
         sudoer: true
 
 docker-manager-first:
-  dev:
-    qa: # @todo set this to be the hostname of your target server
+  hosts:
+    dev: # @todo set this to be the hostname of your target server
       ansible_host: '44.44.44.44' # @todo set this to be the IP address of your server
       data_label: data1 # for manager machines, this should always be "data1"
 

infrastructure/server-setup/playbook.yml

@@ -34,7 +34,9 @@
 
 - hosts: docker-manager-first, docker-workers
   become: yes
-  become_method: sudo
+  become_method: sudo  
+  vars:
+    crontab_user: root
   tasks:
     - include_tasks:
         file: tasks/application.yml

infrastructure/server-setup/production.yml

@@ -50,5 +50,5 @@ docker-workers:
 
 backups:
   hosts:
-    qa:
+    backup: # @todo set this to be the hostname of your backup server
       ansible_host: '66.66.66.66'

infrastructure/server-setup/qa.yml

@@ -28,15 +28,6 @@ all:
           - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABg...Z/rhU= provision@github-runner-243 # example provision user key
           - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABg...Z/rhU= provision@github-runner-244 # example provision user key
 
-    # If the machine is repurposed to also be the backup host, we need to add more keys to the authorized_keys file so that
-    # when the application servers get provisioned, the provision user of this machine can be used.
-    #
-    # @todo remove this key if the machine is not used as a backup host.
-    # Otherwise, add the public key of the private key that's used for the "provision" user on the server of which backups this machine hosts.
-    additional_keys_for_provisioning_user:
-      - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABg...Z/rhU= provision@github-runner-243 # example provision user key
-      - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABg...Z/rhU= provision@github-runner-244 # example provision user key
-
 docker-manager-first:
   hosts:
     qa: # @todo set this to be the hostname of your target server

infrastructure/server-setup/staging.yml

@@ -46,5 +46,5 @@ docker-workers: {}
 # @todo if you do not intend to set up automatic sync from the backup server, you can remove this section.
 backups:
   hosts:
-    qa:
+    backup: # @todo set this to be the hostname of your backup server
       ansible_host: '66.66.66.66' # set this to be the IP address of your backup server

infrastructure/server-setup/tasks/backups/crontab.yml

@@ -16,19 +16,19 @@
     periodic_restore_from_backup: false
   when: periodic_restore_from_backup is not defined
 
-- name: Throw an error if periodic_restore_from_backup is true but restore_backup_encryption_passphrase is not defined
+- name: Throw an error if periodic_restore_from_backup is true but backup_encryption_passphrase is not defined
   fail:
-    msg: 'Error: restore_backup_encryption_passphrase is not defined. It usually means you haven't set RESTORE_BACKUP_ENCRYPTION_PASSPHRASE in your staging environments secrets'
-  when: periodic_restore_from_backup and restore_backup_encryption_passphrase is not defined
+    msg: 'Error: backup_encryption_passphrase is not defined. It usually means you have not set backup_encryption_passphrase in your staging environments secrets'
+  when: periodic_restore_from_backup and backup_encryption_passphrase is not defined
 
 - name: 'Setup crontab to download a backup periodically the opencrvs data'
   cron:
     user: '{{ crontab_user }}'
     name: 'download opencrvs backup'
     minute: '30'
     hour: '0'
-    job: 'cd / && bash /opt/opencrvs/infrastructure/backups/download.sh --passphrase={{ restore_backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --remote_dir={{ external_backup_server_remote_directory }} >> /var/log/opencrvs-restore.log 2>&1'
-    state: "{{ 'present' if (external_backup_server_ip is defined and restore_backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"
+    job: 'cd / && bash /opt/opencrvs/infrastructure/backups/download.sh --passphrase={{ backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --remote_dir={{ external_backup_server_remote_directory }} >> /var/log/opencrvs-restore.log 2>&1'
+    state: "{{ 'present' if (external_backup_server_ip is defined and backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"
 
 - name: 'Setup crontab to restore the opencrvs data'
   cron:
@@ -37,4 +37,4 @@
     minute: '0'
     hour: '1'
     job: 'cd / && bash /opt/opencrvs/infrastructure/backups/restore.sh --replicas=1 >> /var/log/opencrvs-restore.log 2>&1'
-    state: "{{ 'present' if (external_backup_server_ip is defined and restore_backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"
+    state: "{{ 'present' if (external_backup_server_ip is defined and backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"

infrastructure/server-setup/tasks/updates.yml

@@ -25,6 +25,9 @@
     content: |
       Unattended-Upgrade::Package-Blacklist {};
       Unattended-Upgrade::DevRelease "auto";
+      Unattended-Upgrade::Remove-Unused-Kernel-Packages "true";
+      Unattended-Upgrade::Remove-New-Unused-Dependencies "true";
+      Unattended-Upgrade::Remove-Unused-Dependencies "true";
       Unattended-Upgrade::Allowed-Origins {
           "${distro_id}:${distro_codename}-security";
       };

infrastructure/setup-deploy-config.sh

@@ -26,6 +26,19 @@ done
 KIBANA_ENCRYPTION_KEY=`uuidgen`
 sed -i "s/{{KIBANA_ENCRYPTION_KEY}}/$KIBANA_ENCRYPTION_KEY/g" /opt/opencrvs/infrastructure/monitoring/kibana/kibana.yml
 
+# Move metabase file
+mv /opt/opencrvs/infrastructure/metabase.init.db.sql /data/metabase/metabase.init.db.sql
+
+# Replace environment variables from all alert definition files
+for file in /opt/opencrvs/infrastructure/monitoring/elastalert/rules/*.yaml; do
+    sed -i -e "s%{{HOST}}%$1%" $file
+    sed -i -e "s%{{SMTP_HOST}}%$SMTP_HOST%" $file
+    sed -i -e "s%{{SMTP_PORT}}%$SMTP_PORT%" $file
+    sed -i -e "s%{{ALERT_EMAIL}}%$ALERT_EMAIL%" $file
+    sed -i -e "s%{{SENDER_EMAIL_ADDRESS}}%$SENDER_EMAIL_ADDRESS%" $file
+    sed -i -e "s%{{DOMAIN}}%$DOMAIN%" $file
+done
+
 sed -i -e "s%{{MINIO_ROOT_USER}}%$MINIO_ROOT_USER%" /opt/opencrvs/infrastructure/mc-config/config.json
 sed -i -e "s%{{MINIO_ROOT_PASSWORD}}%$MINIO_ROOT_PASSWORD%" /opt/opencrvs/infrastructure/mc-config/config.json
 

src/data-seeding/employees/handler.ts

@@ -8,12 +8,15 @@
  *
  * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
  */
+import { QA_ENV } from '@countryconfig/constants'
 import { readCSVToJSON } from '@countryconfig/utils'
 import { Request, ResponseToolkit } from '@hapi/hapi'
 
 export async function usersHandler(_: Request, h: ResponseToolkit) {
   const users: unknown[] = await readCSVToJSON(
-    './src/data-seeding/employees/source/default-employees.csv'
+    process.env.NODE_ENV === 'production' && !QA_ENV
+      ? './src/data-seeding/employees/source/prod-employees.csv'
+      : './src/data-seeding/employees/source/test-employees.csv'
   )
   return h.response(users)
 }

src/data-seeding/employees/source/prod-employees.csv

@@ -0,0 +1 @@
+primaryOfficeId,givenNames,familyName,systemRole,role,mobile,username,email,password