Merge pull request #89 from RunOnFlux/automate-deployment

feature: add FDM automation

.github/workflows/main.yml

@@ -0,0 +1,42 @@
+name: Deploy FDM configuration
+
+on:
+  push:
+    branches:
+      - main
+  release:
+    types:
+      - created
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+
+    - name: Install Ansible
+      run: pip install ansible
+
+    - name: Set up SSH key
+      env:
+        SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+      run: |
+        mkdir -p ~/.ssh
+        echo "$SSH_PRIVATE_KEY" > private_key
+        chmod 600 private_key
+        echo "Host *" >> ~/.ssh/config
+        echo "  IdentityFile $(pwd)/private_key" >> ~/.ssh/config
+        echo "  StrictHostKeyChecking no" >> ~/.ssh/config
+        echo "  UserKnownHostsFile /dev/null" >> ~/.ssh/config
+
+    - name: Run Ansible playbook
+      env:
+        DEPLOY_ENV: ${{ github.event_name == 'release' && 'production' || 'staging' }}
+      run: |
+        ansible-playbook -i deployment/hosts.ini -e "DEPLOY_ENV=$DEPLOY_ENV" deployment/fdm_setup.yml

config/default.js

@@ -21,7 +21,7 @@ module.exports = {
   },
   mainDomain: 'runonflux.io',
   domainAppType: 'CNAME',
-  emailDomain: '[email protected]',
+  emailDomain: '[email protected]',
   certFolder: 'fluxapps',
   manageCertificateOnly: true,
   automateCertificates: false,

deployment/certbot-cron

@@ -0,0 +1,17 @@
+# /etc/cron.d/certbot: crontab entries for the certbot package
+#
+# Upstream recommends attempting renewal twice a day
+#
+# Eventually, this will be an opportunity to validate certificates
+# haven't been revoked, etc.  Renewal will only occur if expiration
+# is within 30 days.
+#
+# Important Note!  This cronjob will NOT be executed if you are
+# running systemd as your init system.  If you are running systemd,
+# the cronjob.timer function takes precedence over this cronjob.  For
+# more details, see the systemd.timer manpage, or use systemctl show
+# certbot.timer.
+SHELL=/bin/sh
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+0 0 1 * * root bash /opt/update-certs.sh

deployment/default.js.j2

@@ -0,0 +1,57 @@
+const cloudflareConfig = require('./cloudflareConfig');
+const pDNSConfig = require('./PDNSConfig');
+const appsConfig = require('./appsConfig');
+
+module.exports = {
+  server: {
+    port: 16130,
+  },
+  explorer: 'https://explorer.runonflux.io',
+  fallbackexplorer: 'https://explorer.flux.zelcore.io',
+  database: {
+    url: '127.0.0.1',
+    port: 27017,
+    mainDomain: {
+      database: 'runonflux',
+      collections: {
+        // Collection of records associated with domain
+        records: 'records',
+      },
+    },
+  },
+  mainDomain: 'runonflux.io',
+  domainAppType: 'CNAME',
+  emailDomain: '[email protected]',
+  certFolder: 'fluxapps',
+  manageCertificateOnly: {{ manageCertificateOnly }},
+  automateCertificates: false,
+  automateCertificatesForFDMdomains: false,
+  adjustFDMdomains: false,
+  cloudflare: {
+    endpoint: 'https://api.cloudflare.com/client/v4/',
+    apiKey: cloudflareConfig.apiKey,
+    zone: cloudflareConfig.zoneID,
+    domain: cloudflareConfig.domain,
+    manageapp: true,
+    enabled: true,
+  },
+  pDNS: {
+    endpoint: pDNSConfig.apiEndpoint,
+    apiKey: pDNSConfig.apiKey,
+    zone: pDNSConfig.zoneID,
+    domain: pDNSConfig.domain,
+    manageapp: false,
+    enabled: false,
+  },
+  mandatoryApps: appsConfig.mandatoryApps,
+  ownersApps: appsConfig.ownersApps, // Will retrieve only apps of owners specified here
+  whiteListedApps: appsConfig.whiteListedApps, // If there's app in the array, blacklisting will be ignore
+  blackListedApps: appsConfig.blackListedApps,
+  appSubDomain: '{{ appSubDomain }}',
+  fdmAppDomain: '{{ fdmAppDomain }}',
+  useSubset: {{ useSubset }},
+  subset: {
+    start: '{{ startSubset }}',
+    end: '{{ endSubset }}',
+  },
+};

deployment/fdm_setup.yml

@@ -0,0 +1,148 @@
+---
+- name: Install and configure FDM
+  hosts: "{{ 'app2_fdm_servers' if DEPLOY_ENV == 'staging' else 'app_fdm_servers' }}"
+  become: yes
+
+  tasks:
+    - name: Import MongoDB public GPG key
+      ansible.builtin.get_url:
+        url: "https://www.mongodb.org/static/pgp/server-6.0.asc"
+        dest: /etc/apt/keyrings/mongo.asc
+        mode: '0644'
+        force: true
+
+    - name: Add MongoDB repository
+      ansible.builtin.apt_repository:
+        repo: "deb [signed-by=/etc/apt/keyrings/mongo.asc] https://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/6.0 multiverse"
+        state: present
+        filename: mongodb-org-6.0
+
+    - name: Update apt package list
+      ansible.builtin.apt:
+        update_cache: yes
+
+    - name: Install required packages
+      ansible.builtin.package:
+        name:
+          - software-properties-common
+          - certbot
+          - haproxy
+          - dirmngr
+          - gnupg
+          - apt-transport-https
+          - ca-certificates
+          - mongodb-org
+          - build-essential
+          - libssl-dev
+          - curl
+        state: present
+
+    - name: Install Node.js 18.x repository
+      shell: "curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash -"
+
+    - name: Install Node.js
+      apt:
+        name: nodejs
+        state: latest
+        update_cache: yes
+
+    - name: Install PM2 globally using NPM
+      command: npm install -g pm2
+
+    - name: Enable and start MongoDB service
+      ansible.builtin.systemd:
+        name: mongod
+        state: started
+        enabled: yes
+
+    - name: Ensure directory exists
+      ansible.builtin.file:
+        path: /etc/ssl/fluxapps
+        owner: root
+        group: root
+        state: directory
+        mode: '0755'
+
+    - name: Create /opt/update-certs.sh
+      ansible.builtin.file:
+        path: /opt/update-certs.sh
+        state: touch
+        owner: root
+        group: root
+        mode: '0775'
+
+    - name: Copy new certbot-cron
+      ansible.builtin.copy:
+        src: certbot-cron
+        dest: /etc/cron.d/certbot
+        owner: root
+        group: root
+        mode: '0644'
+
+    - name: Clone FDM
+      ansible.builtin.git:
+        repo: https://github.com/RunOnFlux/flux-domain-manager.git
+        dest: flux-domain-manager
+        version: main
+        force: yes
+        update: yes
+        clone: yes
+        depth: 1
+        accept_hostkey: yes
+
+    - name: Clone FDM as CDM
+      ansible.builtin.git:
+        repo: https://github.com/RunOnFlux/flux-domain-manager.git
+        dest: cert-domain-manager
+        version: main
+        update: yes
+        force: yes
+        clone: yes
+        depth: 1
+        accept_hostkey: yes
+
+    - name: Copy default.js config to FDM
+      ansible.builtin.template:
+        src: default.js.j2
+        dest: flux-domain-manager/config/default-0.js
+        mode: '0644'
+      vars:
+        manageCertificateOnly: "false"
+        appSubDomain: "{{ 'app2' if DEPLOY_ENV == 'staging' else 'app' }}"
+        useSubset: "true"
+
+    - name: Copy default.js config to CDM
+      ansible.builtin.template:
+        src: default.js.j2
+        dest: cert-domain-manager/config/default-0.js
+        mode: '0644'
+      vars:
+        manageCertificateOnly: "true"
+        appSubDomain: "{{ 'app2' if DEPLOY_ENV == 'staging' else 'app' }}"
+        useSubset: "false"
+
+    - name: Install Dependencies FDM
+      command: npm install
+      args:
+        chdir: flux-domain-manager
+
+    - name: Install Dependencies CDM
+      command: npm install
+      args:
+        chdir: cert-domain-manager
+
+    - name: start FDM
+      command: sudo pm2 restart FDM
+      args:
+        chdir: flux-domain-manager
+      environment:
+        NODE_ENV: production
+        NODE_CONFIG_DIR: flux-domain-manager/config
+
+    - name: start CDM
+      command: sudo pm2 restart CDM
+      args:
+        chdir: cert-domain-manager
+      environment:
+        NODE_ENV: production
+        NODE_CONFIG_DIR: cert-domain-manager/config

deployment/hosts.ini

@@ -0,0 +1,21 @@
+[app2_fdm_servers]
+fdm_us1_app2 ansible_host=5.161.215.75 ansible_user=root fdmAppDomain=fdm-usa-2-1.runonflux.io startSubset=A endSubset=M
+fdm_us2_app2 ansible_host=5.161.109.34 ansible_user=root fdmAppDomain=fdm-usa-2-2.runonflux.io startSubset=N endSubset=Z
+fdm_fn1_app2 ansible_host=135.181.32.198 ansible_user=root fdmAppDomain=fdm-fn-2-1.runonflux.io startSubset=A endSubset=M
+fdm_fn2_app2 ansible_host=95.216.173.54 ansible_user=root fdmAppDomain=fdm-fn-2-2.runonflux.io startSubset=N endSubset=Z
+
+[app_fdm_servers]
+fdm_fn1_app ansible_host=135.181.99.150 ansible_user=root fdmAppDomain=fdm-fn-1-1.runonflux.io startSubset=A endSubset=G
+fdm_fn2_app ansible_host=65.21.157.205 ansible_user=root fdmAppDomain=fdm-fn-1-2.runonflux.io startSubset=H endSubset=N
+fdm_fn3_app ansible_host=135.181.106.117 ansible_user=root fdmAppDomain=fdm-fn-1-3.runonflux.io startSubset=O endSubset=U
+fdm_fn4_app ansible_host=135.181.103.251 ansible_user=root fdmAppDomain=fdm-fn-1-4.runonflux.io startSubset=V endSubset=Z
+
+fdm_sg1_app ansible_host=146.190.83.190 ansible_user=root fdmAppDomain=fdm-sg-1-1.runonflux.io startSubset=A endSubset=G
+fdm_sg2_app ansible_host=146.190.103.145 ansible_user=root fdmAppDomain=fdm-sg-1-2.runonflux.io startSubset=H endSubset=N
+fdm_sg3_app ansible_host=134.209.107.70 ansible_user=root fdmAppDomain=fdm-sg-1-3.runonflux.io startSubset=O endSubset=U
+fdm_sg4_app ansible_host=146.190.105.10 ansible_user=root fdmAppDomain=fdm-sg-1-4.runonflux.io startSubset=V endSubset=Z
+
+fdm_us1_app ansible_host=5.161.211.14 ansible_user=root fdmAppDomain=fdm-usa-1-1.runonflux.io startSubset=A endSubset=G
+fdm_us2_app ansible_host=5.161.178.20 ansible_user=root fdmAppDomain=fdm-usa-1-2.runonflux.io startSubset=H endSubset=N  
+fdm_us3_app ansible_host=5.161.42.73 ansible_user=root fdmAppDomain=fdm-usa-1-3.runonflux.io startSubset=O endSubset=U  
+fdm_us4_app ansible_host=5.161.81.155 ansible_user=root fdmAppDomain=fdm-usa-1-4.runonflux.io startSubset=V endSubset=Z