deploy.yml

---
# Ansible playbook that deploys NAU.
#
#
# Open edX instance with 3 layers:
# - Load balancer servers
# - Persistence servers (mongo_servers, cache_servers and mysql_servers)
# - Application servers (app_servers & complementary_servers)
#
# NAU also has other services like:
# - richie for marketing site
# - static proxy service
# - fam servers that print course certificate to PDF
#
# Examples:
#   To deploy the application servers and deploy 2 servers at once run:
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit app_servers -e serial_number=2
#
#   To deploy the application servers and migrate the database changes
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit app_servers -e "migrate_db=yes"
#
#   To deploy the application servers with a specific themes version
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit app_servers -e "THEMES_VERSION=master"
#
#   To deploy the complementary servers and its services run:
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit complementary_servers
#
#   To deploy the load balancers run:
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit balancer_servers
#
#   To deploy IdP proxy servers run:
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit idpproxy_servers
#
#   To mark a server for maintenance, like load balancer or mysql server, append to the shell command arguments:
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --tags keepalived --limit slave_persistence_servers -e keepalived_priority=1
#   To mark a server back to normal state, run the same command without the --limit.
#
#   To deploy the docker registry mirror
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit registry_mirror_server -e registrymirror_deploy=true
#
#   To deploy Mongo DB
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit mongo_docker_servers -e mongo_deploy=true
#
#   To deploy Redis cache
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit redis_docker_servers -e redis_deploy=true
#
#   To deploy ElasticSearch cluster
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit elasticsearch_docker_servers -e elasticsearch_deploy=true
#
#   To deploy Richie MySQL
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit richie_mysql_docker_servers -e richie_mysql_deploy=true
#
#   To deploy Richie Application
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit richie_docker_servers -e richie_docker_deploy=true
#
#   To deploy Course Certificate
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit coursecertificate_docker_servers -e coursecertificate_docker_deploy=true
#
#   To deploy StaticProxy
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit staticproxy_docker_servers -e staticproxy_docker_deploy=true
#
#   To deploy Observability
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit observability_docker_servers -e observability_docker_deploy=true
#
#   To deploy Openedx MySQL Database
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit openedx_mysql_docker_servers -e openedx_mysql_deploy=true
#
#   To deploy Openedx Application and build nginx
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit openedx_docker_servers -e openedx_docker_deploy=true -e openedx_nginx_build=true
#
#   To update OpenedX app site configuration
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit openedx_docker_servers -e openedx_app_site_configuration_deploy=true --tags openedx_app_site_configuration
#
#   To deploy Financial Manager MySQL database
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit financial_manager_mysql_docker_servers -e financial_manager_mysql_deploy=true
#
#   To deploy Financial Manager application
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit financial_manager_docker_servers -e financial_manager_deploy=true
#
#   To deploy Percona XtraDB databases
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit xtradb_servers -e xtradb_deploy=true
#
#   To deploy ClickHouse databases
#     ansible-playbook -i nau-data/envs/<env>/hosts.ini deploy.yml --limit clickhouse_servers -e clickhouse_deploy=true
#
# - name: Bootstrap instances of 16.04
#   hosts: all,!idpproxy_servers,!support_server,!build_server
#   gather_facts: no
#   become: true
#   tasks:
#     - import_role:
#         name: python_minimal
#       when: ansible_python_interpreter is not defined

- name: Infrastructure configuration
  hosts: all
  become: True
  gather_facts: True
  tasks:
    - import_role:
        name: users
      when: ansible_distribution == 'Ubuntu'
    - import_role:
        name: ssh_authorized_keys

    - name: On Ubuntu 20.04 or later, you need to install the acl package to becoming non privileged user.
      # Fix of this error:
      #   Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user 
      #   (rc: 1, err: chown: changing ownership of  Operation not permitted
      #   https://github.com/georchestra/ansible/issues/55
      package:
        name: acl
      # need this package so pakiti is installed correctly on NAU
      tags: pakiti
      when: ansible_distribution == 'Ubuntu' and (ansible_distribution_major_version|int) >= 20

    - import_role:
        name: ansible_pakiti_client

    # - name: Install pip
    #   import_role:
    #     name: geerlingguy.pip
    #   when: ansible_distribution == 'Ubuntu' and (ansible_distribution_major_version|int) >= 20

    - name: Configure sysconfig custom values
      sysctl:
        name: "{{ sysconfig_config.name }}"
        value: "{{ sysconfig_config.value }}"
        state: "{{ sysconfig_config.state | default(omit) }}"
        reload: "{{ sysconfig_config.reload | default(omit) }}"
        sysctl_set: "{{ sysconfig_config.sysctl_set | default(omit) }}"
      loop: "{{ sysconfig_configuration | default({}) }}"
      loop_control:
        loop_var: sysconfig_config
        label: "{{ sysconfig_config.name }}"
      tags: sysctl

- name: Bootstrap cnc with Ubuntu 20.04
  hosts: command_and_control_2004
  become: True
  gather_facts: True
  tasks:
    # to run openedx-lms-db-export.yml playbook that runs a script on the a docker container
    - name: Install docker
      import_role:
        name: geerlingguy.docker
      when: ansible_distribution == 'Ubuntu' and (ansible_distribution_major_version|int) >= 20

- name: Configure hosts file
  hosts: all,!support_server,!command_and_control,!build_server
  become: True
  gather_facts: True
  # vars defined on 02_hosts.yml
  roles: 
    - bertvv.hosts
  tags: hosts

- name: Configure load balancer docker
  # Deploy before on backup balancer, so if there is an error during the ansible deployment the primary continue to serve
  hosts: balancer_servers
  serial: "{{ serial_number | default(1) }}"
  become: True
  gather_facts: True
  roles:
    - role: keepalived
      vars: 
        keepalived_priority_override: 1 # Lower priority so the VIPs can swap to other machine.
      when: balancer_keepalived_vrrp_instances is defined
    - role: geerlingguy.docker
    - role: ansible-firewall
    - role: ansible-docker-deploy
    - role: haproxy-netsnmp-perl
    - role: snmpd
    - role: nau_check_urls
    - role: keepalived
      vars:
        keepalived_priority_override: ""
      when: balancer_keepalived_vrrp_instances is defined
  tags: load-balancer

- name: Configure mongo server instance
  hosts: mongo_servers
  become: True
  gather_facts: True
  roles:
    - role: swapfile
    - role: data_dirs
    - role: mongo_3_2
    - role: oraclejdk
    - role: es_kopf

- name: Configure the complementary server group
  hosts: complementary_servers
  serial: "{{ serial_number | default(1) }}"
  become: True
  gather_facts: True
  roles:
    - role: rolling_deploy
      vars:
        rolling_deploy_starting: true
      when: ( groups['complementary_servers'] | length ) > 1
    - role: remove_edx_apt_ppa_repo
    - role: swapfile
    - role: nginx
      nginx_sites:
      - insights
      - edx_notes_api
      - nau_nginx_status
    - role: nginx_default_static_site
      when: use_nginx_default_site
    - role: discovery
      when: COMMON_COMPLEMENTARY_APPLICATION_ROLES is defined and 'discovery' in COMMON_COMPLEMENTARY_APPLICATION_ROLES
    - role: ecommerce
      when: COMMON_COMPLEMENTARY_APPLICATION_ROLES is defined and 'ecommerce' in COMMON_COMPLEMENTARY_APPLICATION_ROLES
    - role: ecomworker
      when: COMMON_COMPLEMENTARY_APPLICATION_ROLES is defined and 'ecommerce' in COMMON_COMPLEMENTARY_APPLICATION_ROLES
    - role: edx_notes_api
      when: COMMON_COMPLEMENTARY_APPLICATION_ROLES is defined and 'edx_notes_api' in COMMON_COMPLEMENTARY_APPLICATION_ROLES
    - role: analytics_api
      when: COMMON_COMPLEMENTARY_APPLICATION_ROLES is defined and 'analytics_api' in COMMON_COMPLEMENTARY_APPLICATION_ROLES
    - role: insights
      when: COMMON_COMPLEMENTARY_APPLICATION_ROLES is defined and 'insights' in COMMON_COMPLEMENTARY_APPLICATION_ROLES
    - role: mfe_deployer
      when: COMMON_COMPLEMENTARY_APPLICATION_ROLES is defined and 'mfe' in COMMON_COMPLEMENTARY_APPLICATION_ROLES
    - role: web_server_monitoring
    - role: snmpd
    - role: nau_check_urls
    - role: rolling_deploy
      vars:
        rolling_deploy_starting: false
      when: ( groups['complementary_servers'] | length ) > 1

- name: Configure pipeline servers
  hosts: pipeline_servers
  become: True
  gather_facts: True
  vars:
    ANALYTICS_API_SERVICE_CONFIG: "When defined, we create the analytics-api users at the db"
  roles:
    - role: remove_edx_apt_ppa_repo
    - role: swapfile
    - role: edxlocal
    # Sometimes we need to comment the next line to run the configuration,
    # because the ensure everything is up and running is broken.
    - analytics_pipeline
    - analytics_pipeline_configuration

# specific variables are configured on nau-data/envs/<env>/group_vars/idpproxy_servers.yml
- name: Configure Shibboleth IdP
  hosts: 
    - idpproxy_servers
  become: True
  gather_facts: True
  tags: 
    - idp
  roles:
    - role: server_files
      vars: 
        server_files: "{{ server_files_before }}"
    - shibboleth-3-4-4-centos7-lite/roles/check-system
    # - shibboleth-3-4-4-centos7-lite/roles/system
    - shibboleth-3-4-4-centos7-lite/roles/httpd
    - shibboleth-3-4-4-centos7-lite/roles/tomcat
    - shibboleth-3-4-4-centos7-lite/roles/postgresql
    - shibboleth-3-4-4-centos7-lite/roles/shibboleth
    - shibboleth-3-4-4-centos7-lite/roles/shibboleth_files
    - role: shibboleth-3-4-4-centos7-lite/roles/cc-cmd
      when: install_cc_cmd is not defined or install_cc_cmd
    - shibboleth-3-4-4-centos7-lite/roles/performance
    - role: server_files
      vars: 
        server_files: "{{ server_files_after }}"

- name: Configure support server
  hosts: support_server
  become: True
  gather_facts: True
  roles:
    - role: geerlingguy.docker
    - role: openark_orchestrator

- name: Configure build server
  hosts: build_server
  tasks:
    - name: Install docker
      import_role:
        name: geerlingguy.docker
    - name: Install Java to run a Jenkins node
      import_role:
        name: geerlingguy.java
    - name: Install pip to build python based software
      import_role:
        name: geerlingguy.pip
    - name: Install virtualenv pip package to build python based software
      pip:
        name: virtualenv
    - name: Install python3.8-venv package to build python based software
      package:
        name: python3.8-venv
    - name: Allow ubuntu user to run docker builds
      user:
        name: ubuntu
        groups: docker
        append: yes
    - name: Install docker_image ansible module dependencies
      pip:
        name: docker-py

- name: Configure docker registry mirror
  hosts: registry_mirror_server
  become: True
  gather_facts: True
  vars:
    _registry_mirror_deploy : "{{ registrymirror_deploy | default(false) | bool }}"
  tasks:
    - name: Deploy registry mirror
      import_role:
        name: registry_mirror_deploy
      when: _registry_mirror_deploy

- name: Deploy Mongo servers
  hosts: mongo_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: mongo_deploy | default(false) | bool

    - import_role:
        name: snmpd
      when: mongo_deploy | default(false) | bool
    - import_role:
        name: snmpd_docker
      when: mongo_deploy | default(false) | bool

    - name: Deploy mongo
      import_role:
        name: mongo_docker_deploy
      vars:
        # mongo_docker_instance_name
        mongo_docker_initdb_root_username: "{{ MONGO_ADMIN_USER | default('admin') if (mongo_docker_init | default(false) | bool) else None }}"
        mongo_docker_initdb_root_password: "{{ MONGO_ADMIN_PASSWORD if (mongo_docker_init | default(false) | bool) else None }}"
        mongo_docker_replSet: "{{ MONGO_REPL_SET }}"
        mongo_docker_keyFile_value: "{{ MONGO_CLUSTER_KEY | default('') }}"
        mongo_docker_command_bind_ip_additional_list:
          - "{{ ansible_additional_ipv4 | default('') }}"
      when: mongo_deploy | default(false) | bool

- name: Set Mongo Feature compatibility
  hosts: mongo_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Get Primary MongoDB Node
      shell: |
        make --no-print-directory --directory {{ mongo_docker_deploy_base_folder }} mongo-primary-node
      register: get_primary_mongodb_node_output
      when: mongo_feature_compatibility_version | default(False)

    - name: Set Mongo Feature compatibility
      shell: |
        docker exec {{ mongo_docker_container_name }} {{ mongo_shell_command }} --username {{ mongo_docker_admin_username }} --password {{ mongo_docker_admin_password }} --eval "db.adminCommand( { setFeatureCompatibilityVersion: '{{ mongo_feature_compatibility_version }}'{% if mongo_feature_compatibility_confirm | default(False) %}, confirm: true{% endif %} } )"
      delegate_to: "{{ get_primary_mongodb_node_output.stdout }}"
      run_once: true
      when: mongo_feature_compatibility_version | default(False)

    - name: Get Mongo Feature compatibility
      shell: |
        docker exec {{ mongo_docker_container_name }} {{ mongo_shell_command }} --username {{ mongo_docker_admin_username }} --password {{ mongo_docker_admin_password }} --eval "JSON.stringify(db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } ))" | tail -n1 | jq -r '.featureCompatibilityVersion.version'
      register: get_mongo_feature_compatibility_version_output
      delegate_to: "{{ get_primary_mongodb_node_output.stdout }}"
      run_once: true
      when: mongo_feature_compatibility_version | default(False)

    - name: Check Mongo Feature compatibility
      fail:
        msg: Incorrect Mongo feature compatibility version, current '{{ get_mongo_feature_compatibility_version_output.stdout }}' expected '{{ mongo_feature_compatibility_version }}'
      delegate_to: "{{ get_primary_mongodb_node_output.stdout }}"
      run_once: true
      when: mongo_feature_compatibility_version | default(False) and get_mongo_feature_compatibility_version_output.stdout != mongo_feature_compatibility_version

    - import_tasks: tasks/healthcheck.yml
      when: mongo_feature_compatibility_version | default(False)

- name: Deploy Elasticsearch servers
  hosts: elasticsearch_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: elasticsearch_deploy | default(false) | bool

    - import_role:
        name: snmpd
      when: elasticsearch_deploy | default(false) | bool
    - import_role:
        name: snmpd_docker
      when: elasticsearch_deploy | default(false) | bool

    - name: Deploy elasticsearch
      import_role:
        name: elasticsearch_docker_deploy
      vars:
        elasticsearch_network_hosts_additional_list:
          - "{{ ansible_additional_ipv4 | default('') }}"
      when: elasticsearch_deploy | default(false) | bool

- name: Deploy cache redis docker servers
  hosts: redis_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Lower keepalived priority to force VIP swap
      import_role:
        name: keepalived
      vars: 
        keepalived_priority_override: 1 # Lower priority so the VIPs can swap to other machine.
      tags: keepalived
      when: ( redis_deploy | default(false) | bool ) and redis_keepalived_vrrp_instances is defined and ( groups['redis_docker_servers'] | length ) > 1

    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: redis_deploy | default(false) | bool

    - import_role:
        name: snmpd
      tags: snmpd
      when: redis_deploy | default(false) | bool
    - import_role:
        name: snmpd_docker
      tags: snmpd
      when: redis_deploy | default(false) | bool

    - name: Deploy cache redis
      import_role:
        name: redis_docker_deploy
      when: redis_deploy | default(false) | bool

    - name: Restore keepalived priority
      import_role:
        name: keepalived
      vars:
        keepalived_priority_override: ""
      tags: keepalived
      when: ( redis_deploy | default(false) | bool ) and redis_keepalived_vrrp_instances is defined and ( groups['redis_docker_servers'] | length ) > 1

- name: Deploy richie mysql docker servers
  hosts: richie_mysql_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Lower keepalived priority to force VIP swap
      import_role:
        name: keepalived
      vars: 
        keepalived_priority_override: 1 # Lower priority so the VIPs can swap to other machine.
      tags: keepalived
      when: ( richie_mysql_deploy | default(false) | bool ) and richie_mysql_keepalived_vrrp_instances is defined and ( groups['richie_mysql_docker_servers'] | length ) > 1

    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: richie_mysql_deploy | default(false) | bool

    - import_role:
        name: snmpd
      tags: snmpd
      when: richie_mysql_deploy | default(false) | bool
    - import_role:
        name: snmpd_docker
      tags: snmpd
      when: richie_mysql_deploy | default(false) | bool

    - name: Deploy richie mysql
      import_role:
        name: mysql_docker_deploy
      vars:
        # Richie shared variables to all environments
        richie_nau_MYSQL_DATABASE: richie_nau
        richie_nau_MYSQL_USER: richie_nau
        richie_mysql_replication_user: richie_replication

        # To fix Richie MySQL authentication
        # use legacy authentication plugin to fix: 
        # django.db.utils.OperationalError: (2059, "Authentication plugin 'caching_sha2_password' 
        # cannot be loaded: /usr/lib/x86_64-linux-gnu/mariadb18/plugin/caching_sha2_password.so: 
        # cannot open shared object file: No such file or directory
        mysql_docker_command_extra: --default-authentication-plugin=mysql_native_password

        # Base configuration for the MySQL instance
        mysql_docker_image: docker.io/mysql:8.0.29
        mysql_docker_replication: true
        mysql_docker_hostname: "{{ richie_mysql_docker_hostname }}"
        mysql_docker_port: "{{ richie_mysql_docker_port }}"
        mysql_docker_instance_name: richie_mysql
        mysql_docker_primary: "{{ richie_mysql_docker_primary }}"
        mysql_root_password: "{{ richie_MYSQL_ROOT_PASSWORD }}"
        mysql_server_id: "{{ groups.richie_mysql_docker_servers.index(inventory_hostname) +1 }}"

        mysql_health_check_user: "{{ richie_mysql_health_check_user }}"
        mysql_health_check_pass: "{{ richie_mysql_health_check_pass | default('') }}"

        # Advanced configuration that assumes that the first server is the primary replica.
        mysql_replication_login_password: "{{ richie_MYSQL_ROOT_PASSWORD }}"
        mysql_replication_master_server_ip: "{{ hostvars[groups.richie_mysql_docker_servers[0]].ansible_host }}"
        mysql_replication_master_port: "{{ hostvars[groups.richie_mysql_docker_servers[0]].richie_mysql_docker_port }}"
        mysql_replication_user: "{{ richie_mysql_replication_user }}"
        mysql_replication_password: "{{ richie_mysql_replication_password }}"
        mysql_replication_init_databases: [ "{{ richie_nau_MYSQL_DATABASE }}" ]
        mysql_replication_additional_databases:
          - "{{ richie_nau_MYSQL_DATABASE | default(None) }}"
        mysql_replication_additional_users:
          # one per site
          - {
              db: "{{ richie_nau_MYSQL_DATABASE | default(None) }}",
              user: "{{ richie_nau_MYSQL_USER | default(None) }}",
              pass: "{{ richie_nau_app_richie_MYSQL_PASSWORD | default(None) }}",
              host: "%"
            }
          - {
              db: "*",
              user: "{{ openark_orchestrator_client_user | default(None) }}",
              pass: "{{ openark_orchestrator_client_password | default(None) }}",
              host: "{{ openark_orchestrator_client_mysql_host }}",
              # ndbinfo.processes:SELECT
              priv: "*.*:SUPER,PROCESS,REPLICATION SLAVE,REPLICATION CLIENT,RELOAD/meta.*:SELECT/performance_schema.replication_group_members:SELECT"
            }
          - {
              db: "*",
              user: "{{ richie_mysql_health_check_user }}",
              host: "{{ richie_mysql_health_check_host | default('%') }}",
            }
      when: richie_mysql_deploy | default(false) | bool

    - name: Restore keepalived priority
      import_role:
        name: keepalived
      vars:
        keepalived_priority_override: ""
      tags: keepalived
      when: ( richie_mysql_deploy | default(false) | bool ) and richie_mysql_keepalived_vrrp_instances is defined and ( groups['richie_mysql_docker_servers'] | length ) > 1

- name: Deploy richie app servers
  hosts: richie_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Start rolling deploy - block load balancer connections
      import_role:
        name: rolling_deploy
      vars:
        rolling_deploy_docker: true
        rolling_deploy_starting: true
      when: ( groups['richie_docker_servers'] | length ) > 1 and ( richie_docker_deploy | default(false) | bool )
    
    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: richie_docker_deploy | default(false) | bool

    - import_role:
        name: snmpd
      when: richie_docker_deploy | default(false) | bool
    - import_role:
        name: snmpd_docker
      when: richie_docker_deploy | default(false) | bool

    - name: Deploy richie as docker compose file
      import_role:
        name: richie_docker_deploy
      vars:
        richie_docker_deploy_run: "{{ richie_docker_deploy | default(false) }}"        
        elasticsearch_cluster_hosts_url_tmp: "{% for elastic_host in groups['elasticsearch_docker_servers'] %}http://{{ hostvars[elastic_host].ansible_host }}:{{ hostvars[elastic_host].elasticsearch_http_port }}{{ ',' if not loop.last else '' }}{% endfor %}"
        richie_docker_deploy_elasticsearch_cluster_hosts_url: "{{ elasticsearch_cluster_hosts_url_tmp.split(',') | list }}"
        # Temporary use elasticsearch VIP at least for the new NAU Richie release
        richie_docker_deploy_redis_host: "{{ hostvars[groups['redis_docker_servers'][0]].redis_virtual_ipv4 }}"
        richie_docker_deploy_redis_port: "{{ hostvars[groups['redis_docker_servers'][0]].redis_docker_port }}"
        richie_docker_deploy_redis_db: "2"
        richie_docker_deploy_mysql_host: "{{ hostvars[groups['richie_mysql_docker_servers'][0]].richie_mysql_virtual_ipv4 }}"
        richie_docker_deploy_mysql_port: "{{ hostvars[groups['richie_mysql_docker_servers'][0]].richie_mysql_docker_port }}"
        # Run database migrations and elasticsearch re-index only on the 1st server
        richie_app_job_deploy: "{{ groups['richie_docker_servers'][0] == inventory_hostname }}"

    - name: End rolling deploy - open load balancer connections
      import_role:
        name: rolling_deploy
      vars:
        rolling_deploy_docker: true
        rolling_deploy_starting: false
      when: ( groups['richie_docker_servers'] | length ) > 1 and ( richie_docker_deploy | default(false) | bool )

- name: Deploy coursecertificate docker servers
  hosts: coursecertificate_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Start rolling deploy - block load balancer connections
      import_role:
        name: rolling_deploy
      vars:
        rolling_deploy_docker: true
        rolling_deploy_starting: true
      when: ( groups['coursecertificate_docker_servers'] | length ) > 1 and ( coursecertificate_docker_deploy | default(false) | bool )
    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: coursecertificate_docker_deploy | default(false) | bool
    - import_role:
        name: snmpd
      tags: snmpd
      when: coursecertificate_docker_deploy | default(false) | bool
    - import_role:
        name: snmpd_docker
      tags: snmpd
      when: coursecertificate_docker_deploy | default(false) | bool
    - name: Deploy coursecertificate as docker compose file
      import_role:
        name: coursecertificate_docker_deploy
      vars:
        coursecertificate_docker_deploy_run: "{{ coursecertificate_docker_deploy | default(false) }}"
    - name: End rolling deploy - open load balancer connections
      import_role:
        name: rolling_deploy
      vars:
        rolling_deploy_docker: true
        rolling_deploy_starting: false
      when: ( groups['coursecertificate_docker_servers'] | length ) > 1 and ( coursecertificate_docker_deploy | default(false) | bool )

- name: Deploy staticproxy docker servers
  hosts: staticproxy_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Start rolling deploy - block load balancer connections
      import_role:
        name: rolling_deploy
      vars:
        rolling_deploy_docker: true
        rolling_deploy_starting: true
      when: ( groups['staticproxy_docker_servers'] | length ) > 1 and ( staticproxy_docker_deploy | default(false) | bool )
    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: staticproxy_docker_deploy | default(false) | bool
    - import_role:
        name: snmpd
      tags: snmpd
      when: staticproxy_docker_deploy | default(false) | bool
    - import_role:
        name: snmpd_docker
      tags: snmpd
      when: staticproxy_docker_deploy | default(false) | bool
    - name: Deploy staticproxy as docker compose file
      import_role:
        name: staticproxy_docker_deploy
      vars:
        staticproxy_docker_deploy_run: "{{ staticproxy_docker_deploy | default(false) }}"
    - name: End rolling deploy - open load balancer connections
      import_role:
        name: rolling_deploy
      vars:
        rolling_deploy_docker: true
        rolling_deploy_starting: false
      when: ( groups['staticproxy_docker_servers'] | length ) > 1 and ( staticproxy_docker_deploy | default(false) | bool )

- name: Deploy openedx mysql docker servers
  hosts: openedx_mysql_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Lower keepalived priority to force VIP swap
      import_role:
        name: keepalived
      vars: 
        keepalived_priority_override: 1 # Lower priority so the VIPs can swap to other machine.
      tags: keepalived
      when: ( openedx_mysql_deploy | default(false) | bool ) and openedx_mysql_read_replica_keepalived_vrrp_instances is defined and ( groups['openedx_mysql_docker_servers'] | length ) > 1

    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: openedx_mysql_deploy | default(false) | bool

    - import_role:
        name: snmpd
      tags: snmpd
      when: openedx_mysql_deploy | default(false) | bool
    - import_role:
        name: snmpd_docker
      tags: snmpd
      when: openedx_mysql_deploy | default(false) | bool

    - name: Deploy openedx mysql
      import_role:
        name: mysql_docker_deploy
      when: openedx_mysql_deploy | default(false) | bool
      vars:
        mysql_docker_image: docker.io/mysql:5.7.44
        mysql_major_release_number: 5

        mysql_docker_configurations_dict_default:
          character_set_server: utf8
          collation_server: utf8_general_ci
          # The number of days for automatic binary log file removal.
          # The default is 0, which means "no automatic removal."
          expire_logs_days: 30
          # If a write to the binary log causes the current log file size to exceed the value of this variable,
          # the server rotates the binary logs (closes the current file and opens the next one).
          # The minimum value is 4096 bytes. The maximum and default value is 1GB.
          max_binlog_size:  100M
          # The maximum permitted number of simultaneous client connections.
          # Default value is 151.
          max_connections:  500

          # Whether to use compression of the source/replica protocol if both source and replica support it.
          slave_compressed_protocol: "ON"

          # The size in bytes of the buffer pool, the memory area where InnoDB caches table and index data.
          # Defaults to 128MB
          innodb_buffer_pool_size: "{{ openedx_mysql_innodb_buffer_pool_size | default('128MB') }}"

          # innodb_flush_log_at_trx_commit: 2

          # In future when on local attached SSDs
          # innodb_io_capacity: 1000

        mysql_docker_configurations_dict_replica:
          # Older DB had: --slave-skip-errors=1032,1062,1451,1452
          slave_skip_errors: 1062,1032

        mysql_docker_configurations_dict: "{{ mysql_docker_configurations_dict_default | combine(mysql_docker_configurations_dict_replica, recursive=True) if not openedx_mysql_docker_primary else mysql_docker_configurations_dict_default }}"

        # The `--max-allowed-packet=1G` is to allow the mysqldump execute a backup. Trying to put it on the mysqldump command directly.
        # The extra command have been removed: `--innodb-buffer-pool-size=4G --innodb-log-file-size=1G --innodb-log-buffer-size=256M`
        # mysql_docker_command_extra: {{ '--slave-skip-errors=1062,1032' if not openedx_mysql_docker_primary else '' }}
        # mysql_replication_login_user: "{{ COMMON_MYSQL_ADMIN_USER }}"

        # Base configuration for the MySQL instance
        mysql_docker_replication: true
        mysql_docker_hostname: "{{ openedx_mysql_docker_hostname }}"
        mysql_docker_port: "{{ openedx_mysql_docker_port }}"
        mysql_docker_instance_name: openedx_mysql
        mysql_docker_primary: "{{ openedx_mysql_docker_primary }}"
        mysql_root_password: "{{ EDXAPP_MYSQL_PASSWORD_ADMIN }}"
        mysql_server_id: "{{ groups.openedx_mysql_docker_servers.index(inventory_hostname) +1 }}"

        mysql_health_check_user: "{{ openedx_mysql_health_check_user | default('health_check_user') }}"
        mysql_health_check_pass: "{{ openedx_mysql_health_check_pass | default('') }}"

        # Advanced configuration that assumes that the first server is the primary replica.
        mysql_replication_login_password: "{{ EDXAPP_MYSQL_PASSWORD_ADMIN }}"
        mysql_replication_master_server_ip: "{{ hostvars[groups.openedx_mysql_docker_servers[0]].ansible_host }}"
        mysql_replication_master_port: "{{ hostvars[groups.openedx_mysql_docker_servers[0]].openedx_mysql_docker_port }}"
        mysql_replication_user: "{{ EDXAPP_MYSQL_REPLICATION_USER }}"
        mysql_replication_password: "{{ EDXAPP_MYSQL_REPLICATION_PASSWORD }}"
        mysql_replication_init_databases: [ edxapp ]
        mysql_replication_additional_databases: "{{ edxlocal_databases }}" # Used by docker_mysql_replication role
        mysql_replication_additional_users: "{{ edxlocal_database_users }}" # Used by docker_mysql_replication role

    - name: Restore keepalived priority
      import_role:
        name: keepalived
      vars:
        keepalived_priority_override: ""
      tags: keepalived
      when: ( openedx_mysql_deploy | default(false) | bool ) and openedx_mysql_read_replica_keepalived_vrrp_instances is defined and ( groups['openedx_mysql_docker_servers'] | length ) > 1

- name: Deploy openedx application docker servers
  hosts: openedx_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Start rolling deploy - block load balancer connections
      import_role:
        name: rolling_deploy
      vars:
        rolling_deploy_docker: true
        rolling_deploy_starting: true
      when: ( groups['openedx_docker_servers'] | length ) > 1 and ( openedx_docker_deploy | default(false) | bool )
    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: openedx_docker_deploy | default(false) | bool
    - import_role:
        name: snmpd
      tags: snmpd
      when: openedx_docker_deploy | default(false) | bool
    - import_role:
        name: snmpd_docker
      tags: snmpd
      when: openedx_docker_deploy | default(false) | bool
    - name: Deploy openedx as docker compose file
      import_role:
        name: openedx_docker_deploy
      vars:
        openedx_docker_deploy_run: "{{ openedx_docker_deploy | default(false) }}"
        elasticsearch_cluster_hosts_url_tmp: "{% for elastic_host in groups['elasticsearch_docker_servers'] %}http://{{ hostvars[elastic_host].ansible_host }}:{{ hostvars[elastic_host].elasticsearch_http_port }}{{ ',' if not loop.last else '' }}{% endfor %}"
        openedx_docker_deploy_elasticsearch_cluster_hosts_url: "{{ elasticsearch_cluster_hosts_url_tmp.split(',') | list }}"
        openedx_docker_deploy_cache_redis_host: "{{ hostvars[groups['redis_docker_servers'][0]].redis_virtual_ipv4 }}" # use VIP
        openedx_docker_deploy_cache_redis_port: "{{ hostvars[groups['redis_docker_servers'][0]].redis_docker_port }}"
        openedx_docker_deploy_celery_redis_host: "{{ hostvars[groups['redis_docker_servers'][0]].ansible_host }}" # don't use VIP, use host IP
        openedx_docker_deploy_celery_redis_port: "{{ hostvars[groups['redis_docker_servers'][0]].redis_docker_port }}"
        openedx_docker_deploy_ecommerce_broker_host: "{{ hostvars[groups['redis_docker_servers'][0]].ansible_host }}" # don't use VIP, use host IP
        openedx_docker_deploy_ecommerce_broker_port: "{{ hostvars[groups['redis_docker_servers'][0]].redis_docker_port }}"
        openedx_docker_deploy_mysql_default_host: "{{ hostvars[groups['openedx_mysql_docker_servers'][0]].ansible_host }}"
        openedx_docker_deploy_mysql_default_port: "{{ hostvars[groups['openedx_mysql_docker_servers'][0]].openedx_mysql_docker_port }}"
        openedx_docker_deploy_mysql_read_replica_host: "{{ hostvars[groups['openedx_mysql_docker_servers'][1]].openedx_mysql_read_replica_virtual_ipv4 | default( hostvars[groups['openedx_mysql_docker_servers'][1]].ansible_host ) }}"
        openedx_docker_deploy_mysql_read_replica_port: "{{ hostvars[groups['openedx_mysql_docker_servers'][1]].openedx_mysql_docker_port }}"
        # s3 to copy ACME issued certificates
        openedx_nginx_s3_certificates_folder: "/acme_certificates"
        openedx_docker_deploy_s3_host: "{{ AWS_S3_DBS_BACKUP_HOST }}"
        openedx_docker_deploy_s3_bucket: "{{ AWS_S3_DBS_BACKUP_BUCKET_NAME }}"
        openedx_docker_deploy_s3_access_key_id: "{{ EDXAPP_BACKUPS_AWS_ACCESS_KEY_ID }}"
        openedx_docker_deploy_s3_secret_access_key: "{{ EDXAPP_BACKUPS_AWS_SECRET_ACCESS_KEY }}"
        # server node where to build the nginx web server image with all static assets inside
        openedx_build_web_server_node: "{{ hostvars[groups['build_server'][0]].ansible_host }}"
    - name: End rolling deploy - open load balancer connections
      import_role:
        name: rolling_deploy
      vars:
        rolling_deploy_docker: true
        rolling_deploy_starting: false
      when: ( groups['openedx_docker_servers'] | length ) > 1 and ( openedx_docker_deploy | default(false) | bool )

    - name: Update openedx site configurations
      import_role:
        name: openedx_app_site_configuration
      when: ( openedx_app_site_configuration_deploy | default(false) | bool )
      run_once: yes
      tags:
        - openedx_app_site_configuration
        - create_or_update_site_configuration

- name: Deploy financial manager mysql docker servers
  hosts: financial_manager_mysql_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: financial_manager_mysql_deploy | default(false) | bool

    - import_role:
        name: snmpd
      tags: snmpd
      when: financial_manager_mysql_deploy | default(false) | bool
    - import_role:
        name: snmpd_docker
      tags: snmpd
      when: financial_manager_mysql_deploy | default(false) | bool

    - name: Deploy financial manager mysql
      import_role:
        name: mysql_docker_deploy
      vars:
        # To fix Financial Manager MySQL authentication
        # use legacy authentication plugin to fix: 
        # django.db.utils.OperationalError: (2059, "Authentication plugin 'caching_sha2_password' 
        # cannot be loaded: /usr/lib/x86_64-linux-gnu/mariadb18/plugin/caching_sha2_password.so: 
        # cannot open shared object file: No such file or directory
        mysql_docker_command_extra: --default-authentication-plugin=mysql_native_password

        # Base configuration for the MySQL instance
        mysql_docker_image: docker.io/mysql:8.0.29
        mysql_major_release_number: 8
        mysql_docker_hostname: "{{ financial_manager_mysql_docker_hostname | default('nau_financial_manager_mysql') }}"
        mysql_docker_port: "{{ financial_manager_mysql_docker_port | default(3306) }}"
        mysql_docker_instance_name: financial_manager_mysql
        mysql_docker_primary: True
        mysql_root_password: "{{ financial_manager_mysql_root_password }}"
        mysql_server_id: "{{ groups.financial_manager_mysql_docker_servers.index(inventory_hostname) +1 }}"

        mysql_health_check_user: "{{ financial_manager_mysql_health_check_user | default('health_check_user') }}"
        mysql_health_check_pass: "{{ financial_manager_mysql_health_check_pass | default('') }}"

        # Advanced configuration that assumes that the first server is the primary replica.
        mysql_replication_login_password: "{{ financial_manager_mysql_root_password }}"
        # mysql_replication_master_server_ip: "{{ hostvars[groups.financial_manager_mysql_docker_servers[0]].ansible_host }}"
        # mysql_replication_master_port: "{{ hostvars[groups.financial_manager_mysql_docker_servers[0]].financial_manager_mysql_docker_port }}"
        # mysql_replication_user: "{{ financial_manager_mysql_replication_user }}"
        # mysql_replication_password: "{{ financial_manager_mysql_replication_password }}"
        mysql_replication_init_databases: [ "{{ financial_manager_mysql_database }}" ]
        mysql_replication_additional_databases:
          - "{{ financial_manager_mysql_database | default(None) }}"
        mysql_replication_additional_users:
          # one per site
          - {
              db: "{{ financial_manager_mysql_database | default(None) }}",
              user: "{{ financial_manager_mysql_user | default(None) }}",
              pass: "{{ financial_manager_mysql_password | default(None) }}",
              host: "%"
            }
          - {
              db: "*",
              user: "{{ financial_manager_mysql_health_check_user | default('health_check_user') }}",
              host: "{{ financial_manager_mysql_health_check_host | default('%') }}",
            }
      when: financial_manager_mysql_deploy | default(false) | bool

- name: Deploy financial manager servers
  hosts: financial_manager_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Start rolling deploy - block load balancer connections
      import_role:
        name: rolling_deploy
      vars:
        rolling_deploy_docker: true
        rolling_deploy_starting: true
      when: ( groups['financial_manager_docker_servers'] | length ) > 1 and ( financial_manager_docker_servers | default(false) | bool )

    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: financial_manager_deploy | default(false) | bool

    - import_role:
        name: snmpd
      tags: snmpd
      when: financial_manager_deploy | default(false) | bool
    - import_role:
        name: snmpd_docker
      tags: snmpd
      when: financial_manager_deploy | default(false) | bool

    - name: Deploy financial manager app
      import_role:
        name: financial_manager_docker_deploy
      when: financial_manager_deploy | default(false) | bool
      vars:
        # MySQL DB
        financial_manager_mysql_docker_hostname: "{{ hostvars[groups['financial_manager_mysql_docker_servers'][0]].ansible_host }}"
        financial_manager_mysql_docker_port: "{{ hostvars[groups['financial_manager_mysql_docker_servers'][0]].financial_manager_mysql_docker_port }}"
        financial_manager_mysql_database: "{{ hostvars[groups['financial_manager_mysql_docker_servers'][0]].financial_manager_mysql_database }}"
        financial_manager_mysql_user: "{{ hostvars[groups['financial_manager_mysql_docker_servers'][0]].financial_manager_mysql_user }}"
        financial_manager_mysql_password: "{{ hostvars[groups['financial_manager_mysql_docker_servers'][0]].financial_manager_mysql_password }}"
        financial_manager_mysql_root_password: "{{ hostvars[groups['financial_manager_mysql_docker_servers'][0]].financial_manager_mysql_root_password }}"

        # Django Cache
        financial_manager_caches_default_redis_host: "{{ hostvars[groups['redis_docker_servers'][0]].redis_virtual_ipv4 }}"
        financial_manager_caches_default_redis_port: "{{ hostvars[groups['redis_docker_servers'][0]].redis_docker_port }}"
        financial_manager_caches_default_redis_db:   "8"

        # Celery broker
        financial_manager_celery_broker_redis_host: "{{ hostvars[groups['redis_docker_servers'][0]].ansible_host }}"
        financial_manager_celery_broker_redis_port: "{{ hostvars[groups['redis_docker_servers'][0]].redis_docker_port }}"
        financial_manager_celery_broker_redis_db:   "9"

    - name: End rolling deploy - open load balancer connections
      import_role:
        name: rolling_deploy
      vars:
        rolling_deploy_docker: true
        rolling_deploy_starting: false
      when: ( groups['financial_manager_docker_servers'] | length ) > 1 and ( financial_manager_docker_servers | default(false) | bool )


- name: Deploy to XtraDB database
  hosts: xtradb_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: xtradb_deploy | default(false) | bool
    - name: Install snmpd
      import_role:
        name: snmpd
      tags: snmpd
      when: xtradb_deploy | default(false) | bool
    - name: Install snmpd docker bridge
      import_role:
        name: snmpd_docker
      tags: snmpd
      when: xtradb_deploy | default(false) | bool
    - name: Deploy Percona XtraDB
      import_role:
        name: xtradb_docker_deploy
      vars:
        # To initialize the xtradb cluster, only 1st node, 1st time with:
        #   -e xtradb_cluster_initialization=true
        xtradb_cluster_hosts_tmp: "{% for host in groups['xtradb_servers'] %}{{ hostvars[host].ansible_host }}{{ ',' if not loop.last else '' }}{% endfor %}"
        xtradb_cluster_hosts: "{{ xtradb_cluster_hosts_tmp.split(',') | list }}"
        xtradb_certs_ca_key: "{{ COMMON_PATH_CUSTOM_FILES }}/xtradb/cert/ca-key.pem"
        xtradb_certs_ca: "{{ COMMON_PATH_CUSTOM_FILES }}/xtradb/cert/ca.pem"
        xtradb_certs_client_cert: "{{ COMMON_PATH_CUSTOM_FILES }}/xtradb/cert/client-cert.pem"
        xtradb_certs_client: "{{ COMMON_PATH_CUSTOM_FILES }}/xtradb/cert/client-key.pem"
        xtradb_certs_private_key: "{{ COMMON_PATH_CUSTOM_FILES }}/xtradb/cert/private_key.pem"
        xtradb_certs_public_key: "{{ COMMON_PATH_CUSTOM_FILES }}/xtradb/cert/public_key.pem"
        xtradb_certs_server_cert: "{{ COMMON_PATH_CUSTOM_FILES }}/xtradb/cert/server-cert.pem"
        xtradb_certs_server_key: "{{ COMMON_PATH_CUSTOM_FILES }}/xtradb/cert/server-key.pem"
        # Prefer the last node in the cluster to be used for state transfer from the cluster.
        # If the list contains a trailing comma, the remaining nodes in the cluster
        # will also be considered if the nodes from the list are not available.
        xtradb_docker_wsrep_sst_donor: "{{ groups['xtradb_servers'][-1] }},"
      when: xtradb_deploy | default(false) | bool


- name: Deploy to ClickHouse database
  hosts: clickhouse_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Install or upgrade docker daemon
      import_role:
        name: geerlingguy.docker
      when: clickhouse_deploy | default(false) | bool
    - name: Install snmpd
      import_role:
        name: snmpd
      tags: snmpd
      when: clickhouse_deploy | default(false) | bool
    - name: Install snmpd docker bridge
      import_role:
        name: snmpd_docker
      tags: snmpd
      when: clickhouse_deploy | default(false) | bool
    - name: Deploy ClickHouse
      import_role:
        name: clickhouse_docker_deploy
      when: clickhouse_deploy | default(false) | bool


- name: Deploy to Kubernetes nodes
  hosts: kubernetes_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - import_role:
        name: snmpd
      tags: snmpd
      when: kubernetes_deploy | default(false) | bool
    - import_role:
        name: snmpd_docker
      tags: snmpd
      when: kubernetes_deploy | default(false) | bool

# Should be the last, because it depends on the docker, and the business projects are the ones that install docker
- name: Deploy observability docker servers
  hosts: observability_docker_servers
  serial: "{{ serial_number | default(1) }}" # deploy in sequence
  become: True
  gather_facts: True
  tasks:
    - name: Deploy observability as docker compose file
      import_role:
        name: observability_docker_deploy
      vars:
        observability_docker_deploy_run: "{{ observability_docker_deploy | default(false) }}"