Merge pull request #144 from SUNET/kano-keydb

Add keydb class

manifests/keydbcluster.pp

@@ -0,0 +1,91 @@
+# A cluster class
+class sunet::keydbcluster(
+  Integer $numnodes = 3,
+  Boolean $hostmode = false,
+  String  $docker_image = 'eqalpha/keydb',
+  String  $docker_tag = 'x86_64_v6.3.4',
+  Optional[Boolean] $tls = false,
+  Optional[String] $cluster_announce_ip = '',
+  Optional[Boolean] $automatic_rectify = false,
+  Optional[Boolean] $prevent_reboot = false,
+)
+{
+
+  # Allow the user to either specify the variable in cosmos-rules or in hiera
+  if $cluster_announce_ip == '' {
+    $__cluster_announce_ip = lookup('cluster_announce_ip', undef, undef, '')
+  } else {
+    $__cluster_announce_ip = $cluster_announce_ip
+  }
+  # Allow the user to use the explicit string ipaddress or ipaddress6 to use the corresponding facts
+  if $__cluster_announce_ip == 'ipaddress' {
+    $_cluster_announce_ip = $facts['ipaddress']
+  } elsif $__cluster_announce_ip == 'ipaddress6' {
+    $_cluster_announce_ip = $facts['ipaddress6']
+  } else {
+    $_cluster_announce_ip = $__cluster_announce_ip
+  }
+
+  $keydb_password = safe_hiera('keydb_password')
+
+  sunet::docker_compose { 'keydbcluster_compose':
+    content          => template('sunet/keydbcluster/docker-compose.yml.erb'),
+    service_name     => 'keydb',
+    compose_dir      => '/opt/',
+    compose_filename => 'docker-compose.yml',
+    description      => 'KeyDB Cluster',
+  }
+  file {'/etc/sysctl.d/55-vm-overcommit.conf':
+    ensure  => present,
+    content => template('sunet/keydbcluster/55-vm-overcommit.conf.erb'),
+  }
+  file {'/opt/keydb-rectify.sh':
+    ensure  => present,
+    mode    => '0755',
+    content => template('sunet/keydbcluster/keydb-rectify.sh.erb'),
+  }
+  if $automatic_rectify {
+    sunet::scriptherder::cronjob { 'keydb-rectify':
+      cmd           => '/opt/keydb-rectify.sh',
+      hour          => '*',
+      minute        => '*/10',
+      ok_criteria   => ['exit_status=0','max_age=2d'],
+      warn_criteria => ['exit_status=1','max_age=3d'],
+    }
+  }
+
+  if $prevent_reboot {
+    include sunet::packages::cowsay
+    file {'/etc/molly-guard/run.d/11-keydbcluster':
+      ensure  => present,
+      mode    => '0755',
+      content => template('sunet/keydbcluster/11-keydbcluster.erb'),
+    }
+  }
+
+  range(0, $numnodes - 1).each |$i|{
+    $clusterportnum = 16379 + $i
+    $keydbportnum = 6379 + $i
+
+    file { "/opt/keydb/node-${i}":
+      ensure  => directory,
+    }
+    -> file { "/opt/keydb/node-${i}/server.conf":
+      ensure  => present,
+      content => template('sunet/keydbcluster/server.conf.erb'),
+    }
+    if $::facts['sunet_nftables_enabled'] == 'yes' or $::facts['dockerhost_advanced_network'] == 'yes' or $::facts['dockerhost2'] == 'yes' {
+      $ports = [$keydbportnum, $clusterportnum]
+      $ports.each|$port| {
+        sunet::nftables::rule { "keydb_port_${port}":
+          rule => "add rule inet filter input tcp dport ${port} counter accept comment \"allow-keydb-${port}\""
+        }
+      }
+    } else {
+      sunet::misc::ufw_allow { "keydb_port_${i}":
+        from => '0.0.0.0/0',
+        port => [$keydbportnum,$clusterportnum],
+      }
+    }
+  }
+}

templates/keydbcluster/11-keydbcluster.erb

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "Checking KeyDB cluster status,please standby..."
+cluster_status=$(redis-cli -a "$(puppet lookup --render-as s keydb_password 2>/dev/null)" --tls --cert /etc/ssl/certs/$(hostname -f)_infra.crt --key /etc/ssl/private/$(hostname -f)_infra.key --cacert /etc/ssl/certs/infra.crt -h $(hostname -f) cluster nodes 2> /dev/null)
+
+my_ip=$(echo "${cluster_status}" | grep myself, | awk '{print $2}' | cut -d : -f 1)
+
+masters_on_host=$(echo "${cluster_status}" | grep "${my_ip}" | grep -c master)
+
+if [ "${masters_on_host}" -gt 1 ]; then
+	echo "WARNING! This machine is master for multiple shards! Maybe do a failover before reboot?" | cowsay
+else
+	echo "KeyDB cluster is ready for reboot"
+fi

templates/keydbcluster/55-vm-overcommit.conf.erb

@@ -0,0 +1 @@
+vm.overcommit_memory = 1

templates/keydbcluster/docker-compose.yml.erb

@@ -0,0 +1,29 @@
+version: '3.2'
+services:
+<% numnodesmone = @numnodes - 1 -%>
+<% for i in 0..numnodesmone -%>
+<% clusterport = 7000 + i %>
+<% keydbport = 6379 + i %>
+<% joinport = 16379 + i %>
+  keydb-node-<%= i %>:
+    container_name: keydb-node-<%= i %>
+    image: <%= @docker_image%>:<%= @docker_tag %>
+    dns:
+      - 89.46.20.75
+      - 89.46.21.29
+      - 89.32.32.32
+<% if @hostmode == true %>
+    network_mode: host
+<% else %>
+    ports:
+      - "<%= keydbport %>:<%= keydbport %>"
+      - "<%= joinport %>:<%= joinport %>"
+<% end %>
+    volumes:
+      - /opt/keydb/node-<%= i %>:/data
+<% if @tls == true -%>
+      - /etc/ssl:/etc/ssl
+<% end %>
+    command: keydb-server /data/server.conf
+    restart: always
+<% end %>

templates/keydbcluster/keydb-rectify.sh.erb

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -e
+
+
+force=0
+case $1 in
+  dryrun)
+    dryrun=1
+    ;;
+  force)
+    force=1
+    ;;
+esac
+
+fqdn=$(hostname -f)
+cert="/etc/ssl/certs/${fqdn}_infra.crt"
+key="/etc/ssl/private/${fqdn}_infra.key"
+ca="/etc/ssl/certs/infra.crt"
+password=$(puppet lookup --render-as s keydb_password 2> /dev/null)
+clusterid=$(redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER MYID 2> /dev/null)
+if (redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER NODES 2> /dev/null| grep -e "^${clusterid}"| grep -q slave); then
+  echo SLAVE, failing over to master
+  if [ "${dryrun}" ]; then
+    echo "No failover will happen in dryrun"
+    exit 0
+  fi
+
+  lock_file="/etc/no-automatic-cosmos"
+  if [ -f "${lock_file}" ] && [ "${force}" -ne 1 ]; then
+    echo "Host is in maintainace mode (by ${lock_file}). No failover will happen."
+  else
+    redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER FAILOVER 2> /dev/null
+  fi
+else
+  echo "Node is MASTER, all is good."
+fi

templates/keydbcluster/server.conf.erb

@@ -0,0 +1,27 @@
+appendonly yes
+bind * -::*
+cluster-config-file nodes.conf
+cluster-enabled yes
+cluster-node-timeout 5000
+#cluster-port <%= @clusterportnum %>
+cluster-slave-validity-factor 0
+cluster-allow-reads-when-down yes
+cluster-require-full-coverage no
+loglevel warning
+<% if @tls%>
+port 0
+tls-port <%= @keydbportnum %>
+tls-replication yes
+tls-cluster yes
+tls-cert-file /etc/ssl/certs/<%= @fqdn %>_infra.crt
+tls-key-file /etc/ssl/private/<%= @fqdn %>_infra.key
+tls-ca-cert-file /etc/ssl/certs/infra.crt
+<% else %>
+port <%= @keydbportnum %>
+<% end %>
+requirepass "<%= @keydb_password %>"
+masterauth "<%= @keydb_password %>"
+<% if @_cluster_announce_ip != ''%>
+cluster-announce-ip <%= @_cluster_announce_ip %>
+<% end %>
+