From 188edf307a707583d6439368394dcd2d8c2f04f8 Mon Sep 17 00:00:00 2001
From: Micke Nordin <kano@sunet.se>
Date: Thu, 21 Mar 2024 11:01:48 +0100
Subject: [PATCH 1/4] Try a KeyDB cluster class, based on the redis cluster
 class

---
 manifests/keydbcluster.pp                     | 91 +++++++++++++++++++
 templates/keydbcluster/11-keydbcluster.erb    | 16 ++++
 .../keydbcluster/55-vm-overcommit.conf.erb    |  1 +
 templates/keydbcluster/docker-compose.yml.erb | 29 ++++++
 templates/keydbcluster/keydb-rectify.sh.erb   | 36 ++++++++
 templates/keydbcluster/server.conf.erb        | 27 ++++++
 6 files changed, 200 insertions(+)
 create mode 100644 manifests/keydbcluster.pp
 create mode 100755 templates/keydbcluster/11-keydbcluster.erb
 create mode 100644 templates/keydbcluster/55-vm-overcommit.conf.erb
 create mode 100644 templates/keydbcluster/docker-compose.yml.erb
 create mode 100644 templates/keydbcluster/keydb-rectify.sh.erb
 create mode 100644 templates/keydbcluster/server.conf.erb

diff --git a/manifests/keydbcluster.pp b/manifests/keydbcluster.pp
new file mode 100644
index 000000000..96b2b4715
--- /dev/null
+++ b/manifests/keydbcluster.pp
@@ -0,0 +1,91 @@
+# A cluster class
+class sunet::keydbcluster(
+  Integer $numnodes = 3,
+  Boolean $hostmode = false,
+  String  $docker_image = 'eqalpha/keydb',
+  String  $docker_tag = 'x86_64_v6.3.4',
+  Optional[Boolean] $tls = false,
+  Optional[String] $cluster_announce_ip = '',
+  Optional[Boolean] $automatic_rectify = false,
+  Optional[Boolean] $prevent_reboot = false,
+)
+{
+
+  # Allow the user to either specify the variable in cosmos-rules or in hiera
+  if $cluster_announce_ip == '' {
+    $__cluster_announce_ip = lookup('cluster_announce_ip', undef, undef, '')
+  } else {
+    $__cluster_announce_ip = $cluster_announce_ip
+  }
+  # Allow the user to use the explicit string ipaddress or ipaddress6 to use the corresponding facts
+  if $__cluster_announce_ip == 'ipaddress' {
+    $_cluster_announce_ip = $facts['ipaddress']
+  } elsif $__cluster_announce_ip == 'ipaddress6' {
+    $_cluster_announce_ip = $facts['ipaddress6']
+  } else {
+    $_cluster_announce_ip = $__cluster_announce_ip
+  }
+
+  $keydb_password = safe_hiera('keydb_password')
+
+  sunet::docker_compose { 'keydbcluster_compose':
+    content          => template('sunet/keydbcluster/docker-compose.yml.erb'),
+    service_name     => 'keydb',
+    compose_dir      => '/opt/',
+    compose_filename => 'docker-compose.yml',
+    description      => 'KeyDB Cluster',
+  }
+  file {'/etc/sysctl.d/55-vm-overcommit.conf':
+    ensure  => present,
+    content => template('sunet/keydbcluster/55-vm-overcommit.conf.erb'),
+  }
+  file {'/opt/keydb-rectify.sh':
+    ensure  => present,
+    mode    => '0755',
+    content => template('sunet/keydbcluster/keydb-rectify.sh.erb'),
+  }
+  if $automatic_rectify {
+    sunet::scriptherder::cronjob { 'keydb-rectify':
+      cmd           => '/opt/keydb-rectify.sh',
+      hour          => '*',
+      minute        => '*/10',
+      ok_criteria   => ['exit_status=0','max_age=2d'],
+      warn_criteria => ['exit_status=1','max_age=3d'],
+    }
+  }
+
+  if $prevent_reboot {
+    include sunet::packages::cowsay
+    file {'/etc/molly-guard/run.d/11-keydbcluster':
+      ensure  => present,
+      mode    => '0755',
+      content => template('sunet/keydbcluster/11-keydbcluster.erb'),
+    }
+  }
+
+  range(0, $numnodes - 1).each |$i|{
+    $clusterportnum = 16379 + $i
+    $keydbportnum = 6379 + $i
+
+    file { "/opt/keydb/node-${i}":
+      ensure  => directory,
+    }
+    -> file { "/opt/keydb/node-${i}/server.conf":
+      ensure  => present,
+      content => template('sunet/keydbcluster/server.conf.erb'),
+    }
+    if $::facts['sunet_nftables_enabled'] == 'yes' or $::facts['dockerhost_advanced_network'] == 'yes' or $::facts['dockerhost2'] == 'yes' {
+      $ports = [$keydbportnum, $clusterportnum]
+      $ports.each|$port| {
+        sunet::nftables::rule { "keydb_port_${port}":
+          rule => "add rule inet filter input tcp dport ${port} counter accept comment \"allow-keydb-${port}\""
+        }
+      }
+    } else {
+      sunet::misc::ufw_allow { "keydb_port_${i}":
+        from => '0.0.0.0/0',
+        port => [$keydbportnum,$clusterportnum],
+      }
+    }
+  }
+}
diff --git a/templates/keydbcluster/11-keydbcluster.erb b/templates/keydbcluster/11-keydbcluster.erb
new file mode 100755
index 000000000..82a6a1f4d
--- /dev/null
+++ b/templates/keydbcluster/11-keydbcluster.erb
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "Checking KeyDB cluster status,please standby..."
+cluster_status=$(redis-cli -a "$(puppet lookup --render-as s keydb_password 2>/dev/null)" --tls --cert /etc/ssl/certs/$(hostname -f)_infra.crt --key /etc/ssl/private/$(hostname -f)_infra.key --cacert /etc/ssl/certs/infra.crt -h $(hostname -f) cluster nodes 2> /dev/null)
+
+my_ip=$(echo "${cluster_status}" | grep myself, | awk '{print $2}' | cut -d : -f 1)
+
+masters_on_host=$(echo "${cluster_status}" | grep "${my_ip}" | grep -c master)
+
+if [ "${masters_on_host}" -gt 1 ]; then
+	echo "WARNING! This machine is master for multiple shards! Maybe do a failover before reboot?" | cowsay
+else
+	echo "KeyDB cluster is ready for reboot"
+fi
diff --git a/templates/keydbcluster/55-vm-overcommit.conf.erb b/templates/keydbcluster/55-vm-overcommit.conf.erb
new file mode 100644
index 000000000..0cf386fb8
--- /dev/null
+++ b/templates/keydbcluster/55-vm-overcommit.conf.erb
@@ -0,0 +1 @@
+vm.overcommit_memory = 1
diff --git a/templates/keydbcluster/docker-compose.yml.erb b/templates/keydbcluster/docker-compose.yml.erb
new file mode 100644
index 000000000..6cfed4eec
--- /dev/null
+++ b/templates/keydbcluster/docker-compose.yml.erb
@@ -0,0 +1,29 @@
+version: '3.2'
+services:
+<% numnodesmone = @numnodes - 1 -%>
+<% for i in 0..numnodesmone -%>
+<% clusterport = 7000 + i %>
+<% keydbport = 6379 + i %>
+<% joinport = 16379 + i %>
+  keydb-node-<%= i %>:
+    container_name: keydb-node-<%= i %>
+    image: <%= @docker_image%>:<%= @docker_tag %>
+    dns:
+      - 89.46.20.75
+      - 89.46.21.29
+      - 89.32.32.32
+<% if @hostmode == true %>
+    network_mode: host
+<% else %>
+    ports:
+      - "<%= keydbport %>:<%= keydbport %>"
+      - "<%= joinport %>:<%= joinport %>"
+<% end %>
+    volumes:
+      - /opt/keydb/node-<%= i %>:/data
+<% if @tls == true -%>
+      - /etc/ssl:/etc/ssl
+<% end %>
+    command: redis-server /data/server.conf
+    restart: always
+<% end %>
diff --git a/templates/keydbcluster/keydb-rectify.sh.erb b/templates/keydbcluster/keydb-rectify.sh.erb
new file mode 100644
index 000000000..7d83af7a1
--- /dev/null
+++ b/templates/keydbcluster/keydb-rectify.sh.erb
@@ -0,0 +1,36 @@
+#!/bin/bash
+set -e
+
+
+force=0
+case $1 in
+  dryrun)
+    dryrun=1
+    ;;
+  force)
+    force=1
+    ;;
+esac
+
+fqdn=$(hostname -f)
+cert="/etc/ssl/certs/${fqdn}_infra.crt"
+key="/etc/ssl/private/${fqdn}_infra.key"
+ca="/etc/ssl/certs/infra.crt"
+password=$(puppet lookup --render-as s keydb_password 2> /dev/null)
+clusterid=$(redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER MYID 2> /dev/null)
+if (redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER NODES 2> /dev/null| grep -e "^${clusterid}"| grep -q slave); then
+  echo SLAVE, failing over to master
+  if [ "${dryrun}" ]; then
+    echo "No failover will happen in dryrun"
+    exit 0
+  fi
+
+  lock_file="/etc/no-automatic-cosmos"
+  if [ -f "${lock_file}" ] && [ "${force}" -ne 1 ]; then
+    echo "Host is in maintainace mode (by ${lock_file}). No failover will happen."
+  else
+    redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER FAILOVER 2> /dev/null
+  fi
+else
+  echo "Node is MASTER, all is good."
+fi
diff --git a/templates/keydbcluster/server.conf.erb b/templates/keydbcluster/server.conf.erb
new file mode 100644
index 000000000..83bb0d7fc
--- /dev/null
+++ b/templates/keydbcluster/server.conf.erb
@@ -0,0 +1,27 @@
+appendonly yes
+bind * -::*
+cluster-config-file nodes.conf
+cluster-enabled yes
+cluster-node-timeout 5000
+cluster-port <%= @clusterportnum %>
+cluster-slave-validity-factor 0
+cluster-allow-reads-when-down yes
+cluster-require-full-coverage no
+loglevel warning
+<% if @tls%>
+port 0
+tls-port <%= @keydbportnum %>
+tls-replication yes
+tls-cluster yes
+tls-cert-file /etc/ssl/certs/<%= @fqdn %>_infra.crt
+tls-key-file /etc/ssl/private/<%= @fqdn %>_infra.key
+tls-ca-cert-file /etc/ssl/certs/infra.crt
+<% else %>
+port <%= @keydbportnum %>
+<% end %>
+requirepass "<%= @keydb_password %>"
+masterauth "<%= @keydb_password %>"
+<% if @_cluster_announce_ip != ''%>
+cluster-announce-ip <%= @_cluster_announce_ip %>
+<% end %>
+

From e4bd1b9645f6209df8669525051e267dbd2b8405 Mon Sep 17 00:00:00 2001
From: Micke Nordin <kano@sunet.se>
Date: Thu, 21 Mar 2024 11:22:08 +0100
Subject: [PATCH 2/4] KeyDB has renamed tooling and binaries as well.

---
 templates/keydbcluster/11-keydbcluster.erb    | 2 +-
 templates/keydbcluster/docker-compose.yml.erb | 2 +-
 templates/keydbcluster/keydb-rectify.sh.erb   | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/templates/keydbcluster/11-keydbcluster.erb b/templates/keydbcluster/11-keydbcluster.erb
index 82a6a1f4d..3c5ac4b3f 100755
--- a/templates/keydbcluster/11-keydbcluster.erb
+++ b/templates/keydbcluster/11-keydbcluster.erb
@@ -3,7 +3,7 @@
 set -e
 
 echo "Checking KeyDB cluster status,please standby..."
-cluster_status=$(redis-cli -a "$(puppet lookup --render-as s keydb_password 2>/dev/null)" --tls --cert /etc/ssl/certs/$(hostname -f)_infra.crt --key /etc/ssl/private/$(hostname -f)_infra.key --cacert /etc/ssl/certs/infra.crt -h $(hostname -f) cluster nodes 2> /dev/null)
+cluster_status=$(keydb-cli -a "$(puppet lookup --render-as s keydb_password 2>/dev/null)" --tls --cert /etc/ssl/certs/$(hostname -f)_infra.crt --key /etc/ssl/private/$(hostname -f)_infra.key --cacert /etc/ssl/certs/infra.crt -h $(hostname -f) cluster nodes 2> /dev/null)
 
 my_ip=$(echo "${cluster_status}" | grep myself, | awk '{print $2}' | cut -d : -f 1)
 
diff --git a/templates/keydbcluster/docker-compose.yml.erb b/templates/keydbcluster/docker-compose.yml.erb
index 6cfed4eec..4f20d2fd4 100644
--- a/templates/keydbcluster/docker-compose.yml.erb
+++ b/templates/keydbcluster/docker-compose.yml.erb
@@ -24,6 +24,6 @@ services:
 <% if @tls == true -%>
       - /etc/ssl:/etc/ssl
 <% end %>
-    command: redis-server /data/server.conf
+    command: keydb-server /data/server.conf
     restart: always
 <% end %>
diff --git a/templates/keydbcluster/keydb-rectify.sh.erb b/templates/keydbcluster/keydb-rectify.sh.erb
index 7d83af7a1..38c72ad37 100644
--- a/templates/keydbcluster/keydb-rectify.sh.erb
+++ b/templates/keydbcluster/keydb-rectify.sh.erb
@@ -17,8 +17,8 @@ cert="/etc/ssl/certs/${fqdn}_infra.crt"
 key="/etc/ssl/private/${fqdn}_infra.key"
 ca="/etc/ssl/certs/infra.crt"
 password=$(puppet lookup --render-as s keydb_password 2> /dev/null)
-clusterid=$(redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER MYID 2> /dev/null)
-if (redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER NODES 2> /dev/null| grep -e "^${clusterid}"| grep -q slave); then
+clusterid=$(keydb-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER MYID 2> /dev/null)
+if (keydb-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER NODES 2> /dev/null| grep -e "^${clusterid}"| grep -q slave); then
   echo SLAVE, failing over to master
   if [ "${dryrun}" ]; then
     echo "No failover will happen in dryrun"
@@ -29,7 +29,7 @@ if (redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}
   if [ -f "${lock_file}" ] && [ "${force}" -ne 1 ]; then
     echo "Host is in maintainace mode (by ${lock_file}). No failover will happen."
   else
-    redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER FAILOVER 2> /dev/null
+    keydb-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER FAILOVER 2> /dev/null
   fi
 else
   echo "Node is MASTER, all is good."

From 5e03ad6bfeab974e33349fe1f726c0036a3c6958 Mon Sep 17 00:00:00 2001
From: Micke Nordin <kano@sunet.se>
Date: Thu, 21 Mar 2024 11:27:57 +0100
Subject: [PATCH 3/4] We still rely on redis-cli from the host though

---
 templates/keydbcluster/11-keydbcluster.erb  | 2 +-
 templates/keydbcluster/keydb-rectify.sh.erb | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/templates/keydbcluster/11-keydbcluster.erb b/templates/keydbcluster/11-keydbcluster.erb
index 3c5ac4b3f..82a6a1f4d 100755
--- a/templates/keydbcluster/11-keydbcluster.erb
+++ b/templates/keydbcluster/11-keydbcluster.erb
@@ -3,7 +3,7 @@
 set -e
 
 echo "Checking KeyDB cluster status,please standby..."
-cluster_status=$(keydb-cli -a "$(puppet lookup --render-as s keydb_password 2>/dev/null)" --tls --cert /etc/ssl/certs/$(hostname -f)_infra.crt --key /etc/ssl/private/$(hostname -f)_infra.key --cacert /etc/ssl/certs/infra.crt -h $(hostname -f) cluster nodes 2> /dev/null)
+cluster_status=$(redis-cli -a "$(puppet lookup --render-as s keydb_password 2>/dev/null)" --tls --cert /etc/ssl/certs/$(hostname -f)_infra.crt --key /etc/ssl/private/$(hostname -f)_infra.key --cacert /etc/ssl/certs/infra.crt -h $(hostname -f) cluster nodes 2> /dev/null)
 
 my_ip=$(echo "${cluster_status}" | grep myself, | awk '{print $2}' | cut -d : -f 1)
 
diff --git a/templates/keydbcluster/keydb-rectify.sh.erb b/templates/keydbcluster/keydb-rectify.sh.erb
index 38c72ad37..7d83af7a1 100644
--- a/templates/keydbcluster/keydb-rectify.sh.erb
+++ b/templates/keydbcluster/keydb-rectify.sh.erb
@@ -17,8 +17,8 @@ cert="/etc/ssl/certs/${fqdn}_infra.crt"
 key="/etc/ssl/private/${fqdn}_infra.key"
 ca="/etc/ssl/certs/infra.crt"
 password=$(puppet lookup --render-as s keydb_password 2> /dev/null)
-clusterid=$(keydb-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER MYID 2> /dev/null)
-if (keydb-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER NODES 2> /dev/null| grep -e "^${clusterid}"| grep -q slave); then
+clusterid=$(redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER MYID 2> /dev/null)
+if (redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER NODES 2> /dev/null| grep -e "^${clusterid}"| grep -q slave); then
   echo SLAVE, failing over to master
   if [ "${dryrun}" ]; then
     echo "No failover will happen in dryrun"
@@ -29,7 +29,7 @@ if (keydb-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}
   if [ -f "${lock_file}" ] && [ "${force}" -ne 1 ]; then
     echo "Host is in maintainace mode (by ${lock_file}). No failover will happen."
   else
-    keydb-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER FAILOVER 2> /dev/null
+    redis-cli -a "${password}" -h "${fqdn}" --tls --cert "${cert}" --key "${key}" --cacert "${ca}" CLUSTER FAILOVER 2> /dev/null
   fi
 else
   echo "Node is MASTER, all is good."

From 7f298463ac93aa733287299767c66f8f50a0816d Mon Sep 17 00:00:00 2001
From: Micke Nordin <kano@sunet.se>
Date: Thu, 21 Mar 2024 11:48:25 +0100
Subject: [PATCH 4/4] cluster-port does not exist in keydb 6.x

---
 templates/keydbcluster/server.conf.erb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/keydbcluster/server.conf.erb b/templates/keydbcluster/server.conf.erb
index 83bb0d7fc..1469b7530 100644
--- a/templates/keydbcluster/server.conf.erb
+++ b/templates/keydbcluster/server.conf.erb
@@ -3,7 +3,7 @@ bind * -::*
 cluster-config-file nodes.conf
 cluster-enabled yes
 cluster-node-timeout 5000
-cluster-port <%= @clusterportnum %>
+#cluster-port <%= @clusterportnum %>
 cluster-slave-validity-factor 0
 cluster-allow-reads-when-down yes
 cluster-require-full-coverage no