Additional redis write (#2304)

This PR:

* allows to configure gundeck to write to an additional redis (cluster or master mode); optionally. Reads will happen from the main redis.
* adds a test for the redis-migration scenario: configure to write to two redises (old and new); insert some data; configure only the new redis as default redis, read, ensure data is present.
* adds redis-cluster in docker-compose for local integration tests (a little more CPU/RAM needed)
* adds redis-cluster wrapper helm chart for CI/kubernetes-based integration tests (and more generally also for on-premise deployments). redis-cluster requires dynamic persistent volume provisioning present on the cluster; which was introduced in our kube-ci cluster with zinfra/cailleach#1037

Drive-by refactoring change (not strictly needed on this PR):

* [Move cannon's internal API type to wire-api](7ace63f)

Related to zinfra/cailleach#1021

This is needed to perform a smooth no-downtime migration from one redis to another as detailed in https://github.com/zinfra/cailleach/blob/master/targets/wire/notes-redis-migration.md#how-to-migrate-traffic-from-redis-in-ec2classic-to-redis-in-eks

Co-authored-by: Akshay Mankar <[email protected]>

Makefile

@@ -8,13 +8,13 @@ DOCKER_TAG            ?= $(USER)
 # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
 HELM_SEMVER           ?= 0.0.42
 # The list of helm charts needed on internal kubernetes testing environments
-CHARTS_INTEGRATION    := wire-server databases-ephemeral fake-aws nginx-ingress-controller nginx-ingress-services wire-server-metrics fluent-bit kibana sftd restund coturn
+CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster fake-aws nginx-ingress-controller nginx-ingress-services wire-server-metrics fluent-bit kibana sftd restund coturn
 # The list of helm charts to publish on S3
 # FUTUREWORK: after we "inline local subcharts",
 # (e.g. move charts/brig to charts/wire-server/brig)
 # this list could be generated from the folder names under ./charts/ like so:
 # CHARTS_RELEASE := $(shell find charts/ -maxdepth 1 -type d | xargs -n 1 basename | grep -v charts)
-CHARTS_RELEASE        := wire-server redis-ephemeral databases-ephemeral fake-aws fake-aws-s3 fake-aws-sqs aws-ingress  fluent-bit kibana backoffice calling-test demo-smtp elasticsearch-curator elasticsearch-external elasticsearch-ephemeral minio-external cassandra-external nginx-ingress-controller nginx-ingress-services reaper wire-server-metrics sftd restund coturn
+CHARTS_RELEASE        := wire-server redis-ephemeral redis-cluster databases-ephemeral fake-aws fake-aws-s3 fake-aws-sqs aws-ingress  fluent-bit kibana backoffice calling-test demo-smtp elasticsearch-curator elasticsearch-external elasticsearch-ephemeral minio-external cassandra-external nginx-ingress-controller nginx-ingress-services reaper wire-server-metrics sftd restund coturn
 BUILDAH_PUSH          ?= 0
 KIND_CLUSTER_NAME     := wire-server
 BUILDAH_KIND_LOAD     ?= 1

changelog.d/5-internal/redis-migration

@@ -0,0 +1 @@
+Allow specifying 'redisAdditionalWrite' for a secondary redis to which gundeck will write in the context of a redis migration without downtime.

charts/gundeck/templates/configmap.yaml

@@ -26,6 +26,11 @@ data:
       port: {{ .redis.port }}
       connectionMode: {{ .redis.connectionMode }}
 
+    {{- if .redisAdditionalWrite }}
+    redisAdditionalWrite:
+      {{- toYaml .redisAdditionalWrite | nindent 6 }}
+    {{- end }}
+
     # Gundeck uses discovery for AWS access key / secrets
     # For more details, check amazonka's documentation at:
     # https://hackage.haskell.org/package/amazonka-1.4.5/docs/Network-AWS.html#t:Credentials

charts/gundeck/templates/tests/configmap.yaml

@@ -35,3 +35,9 @@ data:
     brig:
       host: brig
       port: 8080
+
+    # a "redis migration" test in gundeck makes use of a second (distinct) redis
+    redis2:
+      host: redis-ephemeral-master
+      port: 6379
+      connectionMode: master

charts/gundeck/values.yaml

@@ -20,6 +20,10 @@ config:
     host: redis-ephemeral-master
     port: 6379
     connectionMode: "master" # master | cluster
+  # redisAdditionalWrite:
+  #   host: redis-two
+  #   port: 6379
+  #   connectionMode: master
   bulkPush: true
   aws:
     region: "eu-west-1"

charts/redis-cluster/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+description: Wrapper chart for bitnami/redis-cluster
+name: redis-cluster
+version: 0.0.42

charts/redis-cluster/requirements.yaml

@@ -0,0 +1,4 @@
+dependencies:
+- name: redis-cluster
+  version: 7.4.8
+  repository: https://charts.bitnami.com/bitnami

charts/redis-cluster/values.yaml

@@ -0,0 +1,3 @@
+redis-cluster:
+  usePassword: false
+  fullnameOverride: "redis-cluster"

deploy/dockerephemeral/docker-compose.yaml

@@ -3,6 +3,12 @@ networks:
   demo_wire:
     external: false
 
+  redis:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.20.0.0/24
+
 services:
   fake_dynamodb:
     container_name: demo_wire_dynamodb
@@ -72,6 +78,86 @@ services:
     networks:
       - demo_wire
 
+  redis-cluster:
+    image: 'redis:6.0-alpine'
+    command: redis-cli --cluster create 172.20.0.31:6373 172.20.0.32:6374 172.20.0.33:6375 172.20.0.34:6376 172.20.0.35:6377 172.20.0.36:6378 --cluster-replicas 1 --cluster-yes
+    networks:
+      redis:
+        ipv4_address: 172.20.0.30
+    depends_on:
+      - redis-node-1
+      - redis-node-2
+      - redis-node-3
+      - redis-node-4
+      - redis-node-5
+      - redis-node-6
+  redis-node-1:
+    image: 'redis:6.0-alpine'
+    command: redis-server /usr/local/etc/redis/redis.conf
+    ports:
+      - '127.0.0.1:6373:6373'
+    volumes:
+      - redis-node-1-data:/var/lib/redis
+      - ./docker/redis-node-1.conf:/usr/local/etc/redis/redis.conf
+    networks:
+      redis:
+        ipv4_address: 172.20.0.31
+  redis-node-2:
+    image: 'redis:6.0-alpine'
+    command: redis-server /usr/local/etc/redis/redis.conf
+    ports:
+      - '127.0.0.1:6374:6374'
+    volumes:
+      - redis-node-2-data:/var/lib/redis
+      - ./docker/redis-node-2.conf:/usr/local/etc/redis/redis.conf
+    networks:
+      redis:
+        ipv4_address: 172.20.0.32
+  redis-node-3:
+    image: 'redis:6.0-alpine'
+    command: redis-server /usr/local/etc/redis/redis.conf
+    ports:
+      - '127.0.0.1:6375:6375'
+    volumes:
+      - redis-node-3-data:/var/lib/redis
+      - ./docker/redis-node-3.conf:/usr/local/etc/redis/redis.conf
+    networks:
+      redis:
+        ipv4_address: 172.20.0.33
+  redis-node-4:
+    image: 'redis:6.0-alpine'
+    command: redis-server /usr/local/etc/redis/redis.conf
+    ports:
+      - '127.0.0.1:6376:6376'
+    volumes:
+      - redis-node-4-data:/var/lib/redis
+      - ./docker/redis-node-4.conf:/usr/local/etc/redis/redis.conf
+    networks:
+      redis:
+        ipv4_address: 172.20.0.34
+  redis-node-5:
+    image: 'redis:6.0-alpine'
+    command: redis-server /usr/local/etc/redis/redis.conf
+    ports:
+      - '127.0.0.1:6377:6377'
+    volumes:
+      - redis-node-5-data:/var/lib/redis
+      - ./docker/redis-node-5.conf:/usr/local/etc/redis/redis.conf
+    networks:
+      redis:
+        ipv4_address: 172.20.0.35
+  redis-node-6:
+    image: 'redis:6.0-alpine'
+    command: redis-server /usr/local/etc/redis/redis.conf
+    ports:
+      - '127.0.0.1:6378:6378'
+    volumes:
+      - redis-node-6-data:/var/lib/redis
+      - ./docker/redis-node-6.conf:/usr/local/etc/redis/redis.conf
+    networks:
+      redis:
+        ipv4_address: 172.20.0.36
+
   elasticsearch:
     container_name: demo_wire_elasticsearch
     #image: elasticsearch:5.6
@@ -202,3 +288,11 @@ services:
     ports:
       - "9053:53"
       - "9053:53/udp"
+
+volumes:
+  redis-node-1-data:
+  redis-node-2-data:
+  redis-node-3-data:
+  redis-node-4-data:
+  redis-node-5-data:
+  redis-node-6-data:

deploy/dockerephemeral/docker/redis-node-1.conf

@@ -0,0 +1,5 @@
+port 6373
+cluster-enabled yes
+cluster-config-file nodes.conf
+cluster-node-timeout 5000
+appendonly yes

deploy/dockerephemeral/docker/redis-node-2.conf

@@ -0,0 +1,5 @@
+port 6374
+cluster-enabled yes
+cluster-config-file nodes.conf
+cluster-node-timeout 5000
+appendonly yes

deploy/dockerephemeral/docker/redis-node-3.conf

@@ -0,0 +1,5 @@
+port 6375
+cluster-enabled yes
+cluster-config-file nodes.conf
+cluster-node-timeout 5000
+appendonly yes

deploy/dockerephemeral/docker/redis-node-4.conf

@@ -0,0 +1,5 @@
+port 6376
+cluster-enabled yes
+cluster-config-file nodes.conf
+cluster-node-timeout 5000
+appendonly yes

deploy/dockerephemeral/docker/redis-node-5.conf

@@ -0,0 +1,5 @@
+port 6377
+cluster-enabled yes
+cluster-config-file nodes.conf
+cluster-node-timeout 5000
+appendonly yes

deploy/dockerephemeral/docker/redis-node-6.conf

@@ -0,0 +1,5 @@
+port 6378
+cluster-enabled yes
+cluster-config-file nodes.conf
+cluster-node-timeout 5000
+appendonly yes

hack/bin/integration-setup-federation.sh

@@ -19,7 +19,7 @@ ${DIR}/integration-cleanup.sh
 # script beforehand on all relevant charts to download the nested dependencies
 # (e.g. cassandra from underneath databases-ephemeral)
 echo "updating recursive dependencies ..."
-charts=(fake-aws databases-ephemeral wire-server nginx-ingress-controller nginx-ingress-services)
+charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
 for chart in "${charts[@]}"; do
     "$DIR/update.sh" "$CHARTS_DIR/$chart"
 done

hack/bin/integration-setup.sh

@@ -13,7 +13,7 @@ CHARTS_DIR="${TOP_LEVEL}/.local/charts"
 "${DIR}/integration-cleanup.sh"
 
 echo "updating recursive dependencies ..."
-charts=(fake-aws databases-ephemeral wire-server nginx-ingress-controller nginx-ingress-services)
+charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
 for chart in "${charts[@]}"; do
     "$DIR/update.sh" "$CHARTS_DIR/$chart"
 done

hack/helm_vars/redis-cluster/values.yaml

@@ -0,0 +1,8 @@
+global:
+  storageClass: csi-hostpath-sc
+
+redis-cluster:
+  persistence:
+    size: 100Mi
+  volumePermissions:
+    enabled: true

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -179,7 +179,8 @@ gundeck:
       host: cassandra-ephemeral
       replicaCount: 1
     redis:
-      host: redis-ephemeral-master
+      host: redis-cluster
+      connectionMode: cluster
     aws:
       account: "123456789012"
       region: eu-west-1

hack/helmfile-single.yaml

@@ -19,6 +19,9 @@ repositories:
   - name: stable
     url: 'https://charts.helm.sh/stable'
 
+  - name: bitnami
+    url: 'https://charts.bitnami.com/bitnami'
+
 releases:
   - name: '{{ .Values.namespace }}-fake-aws'
     namespace: '{{ .Values.namespace }}'
@@ -30,6 +33,12 @@ releases:
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/databases-ephemeral'
 
+  - name: '{{ .Values.namespace }}-redis-cluster'
+    namespace: '{{ .Values.namespace }}'
+    chart: '../.local/charts/redis-cluster'
+    values:
+      - './helm_vars/redis-cluster/values.yaml'
+
   - name: '{{ .Values.namespace }}-nginx-ingress-controller'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/nginx-ingress-controller'

hack/helmfile.yaml

@@ -31,6 +31,9 @@ repositories:
   - name: stable
     url: 'https://charts.helm.sh/stable'
 
+  - name: bitnami
+    url: 'https://charts.bitnami.com/bitnami'
+
 releases:
   - name: '{{ .Values.namespace }}-fake-aws'
     namespace: '{{ .Values.namespace }}'
@@ -52,6 +55,18 @@ releases:
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/databases-ephemeral'
 
+  - name: '{{ .Values.namespace }}-redis-cluster'
+    namespace: '{{ .Values.namespace }}'
+    chart: '../.local/charts/redis-cluster'
+    values:
+      - './helm_vars/redis-cluster/values.yaml'
+
+  - name: '{{ .Values.namespace }}-redis-cluster-2'
+    namespace: '{{ .Values.namespaceFed2 }}'
+    chart: '../.local/charts/redis-cluster'
+    values:
+      - './helm_vars/redis-cluster/values.yaml'
+
   - name: '{{ .Values.namespace }}-nginx-ingress-controller'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/nginx-ingress-controller'

libs/gundeck-types/gundeck-types.cabal

@@ -4,7 +4,7 @@ cabal-version: 1.12
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: 5ecbff5b5248085ccaf245621fb9388ff13a1cdace8dff356ea98ccd1cb72a03
+-- hash: 738b7369cd6cbbf07382a5645aa9c475292f016888ee76c40569aa56d24d9cae
 
 name:           gundeck-types
 version:        1.45.0
@@ -20,10 +20,8 @@ build-type:     Simple
 library
   exposed-modules:
       Gundeck.Types
-      Gundeck.Types.BulkPush
       Gundeck.Types.Common
       Gundeck.Types.Event
-      Gundeck.Types.Notification
       Gundeck.Types.Presence
       Gundeck.Types.Push
       Gundeck.Types.Push.V2

libs/gundeck-types/src/Gundeck/Types.hs

@@ -21,6 +21,5 @@ module Gundeck.Types
 where
 
 import Gundeck.Types.Event as G
-import Gundeck.Types.Notification as G
 import Gundeck.Types.Presence as G
 import Gundeck.Types.Push as G

libs/tasty-cannon/package.yaml

@@ -16,7 +16,6 @@ dependencies:
 - bytestring
 - bytestring-conversion
 - data-timeout
-- gundeck-types
 - exceptions
 - http-client >=0.5
 - http-types
@@ -25,5 +24,6 @@ dependencies:
 - tasty-hunit >=0.9
 - types-common >=0.16
 - websockets >=0.8
+- wire-api
 library:
   source-dirs: src

libs/tasty-cannon/src/Test/Tasty/Cannon.hs

@@ -83,14 +83,14 @@ import Data.ByteString.Conversion
 import Data.Id
 import Data.List1
 import Data.Timeout (Timeout, TimeoutUnit (..), (#))
-import Gundeck.Types
 import Imports
 import Network.HTTP.Client
 import qualified Network.HTTP.Client as Http
 import Network.HTTP.Types.Status
 import qualified Network.WebSockets as WS
 import System.Random (randomIO)
 import Test.Tasty.HUnit
+import Wire.API.Internal.Notification
 
 type Cannon = Http.Request -> Http.Request
 

libs/tasty-cannon/tasty-cannon.cabal

@@ -4,7 +4,7 @@ cabal-version: 1.12
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: 33f3a557e5296946dc5929fd741dfa0bb68e83484e1760cd425204e7defec8af
+-- hash: be26c1ad508de205143282797883e24b2b8ba5dad64b576df52e8c2665bb3c32
 
 name:           tasty-cannon
 version:        0.4.0
@@ -73,12 +73,12 @@ library
     , bytestring-conversion
     , data-timeout
     , exceptions
-    , gundeck-types
     , http-client >=0.5
     , http-types
     , imports
     , random
     , tasty-hunit >=0.9
     , types-common >=0.16
     , websockets >=0.8
+    , wire-api
   default-language: Haskell2010