add multi-master option to galera

This change adds the new value enableMultiMaster to the
Galera CR defaulting to false; when set to true, the
"statefulset.kubernetes.io/pod-name" key is removed
from the mariadb service spec, and when set to false it is
restored.  This has the effect of publishing the service either
under one specific pod for active/passive use, or using
any pod for any request which is multi-master use.  the value
of the attribute is also propigated out to the status within
the MariaDBDatabase CR, which is then consumable by openstack
services in a similar manner as the TLS settings.   Openstack
services will then want to set wsrep_sync_wait accordingly
for AP or MM mode, which will be proposed in followup changes.

Fixes: OSPRH-7406

api/bases/mariadb.openstack.org_galeras.yaml

@@ -64,6 +64,10 @@ spec:
                 description: When TLS is configured, only allow connections to the
                   DB over TLS
                 type: boolean
+              enableMultiMaster:
+                default: false
+                description: Use multi-master service routing
+                type: boolean
               logToDisk:
                 description: Log Galera pod's output to disk
                 type: boolean
@@ -104,6 +108,7 @@ spec:
                 type: object
             required:
             - containerImage
+            - enableMultiMaster
             - replicas
             - secret
             - storageClass

api/bases/mariadb.openstack.org_mariadbdatabases.yaml

@@ -98,6 +98,9 @@ spec:
                   - type
                   type: object
                 type: array
+              enableMultiMaster:
+                description: whether the DB instance is using multi-master routing
+                type: boolean
               hash:
                 additionalProperties:
                   type: string

api/v1beta1/galera_types.go

@@ -89,6 +89,9 @@ type GaleraSpecCore struct {
 	// +kubebuilder:validation:Optional
 	// Log Galera pod's output to disk
 	LogToDisk bool `json:"logToDisk"`
+	// +kubebuilder:default=false
+	// Use multi-master service routing
+	EnableMultiMaster bool `json:"enableMultiMaster"`
 }
 
 // GaleraAttributes holds startup information for a Galera host

api/v1beta1/mariadbdatabase_types.go

@@ -55,6 +55,9 @@ type MariaDBDatabaseStatus struct {
 
 	// Whether TLS is supported by the DB instance
 	TLSSupport bool `json:"tlsSupport,omitempty"`
+
+	// whether the DB instance is using multi-master routing
+	EnableMultiMaster bool `json:"enableMultiMaster,omitempty"`
 }
 
 //+kubebuilder:object:root=true

config/crd/bases/mariadb.openstack.org_galeras.yaml

@@ -64,6 +64,10 @@ spec:
                 description: When TLS is configured, only allow connections to the
                   DB over TLS
                 type: boolean
+              enableMultiMaster:
+                default: false
+                description: Use multi-master service routing
+                type: boolean
               logToDisk:
                 description: Log Galera pod's output to disk
                 type: boolean
@@ -104,6 +108,7 @@ spec:
                 type: object
             required:
             - containerImage
+            - enableMultiMaster
             - replicas
             - secret
             - storageClass

config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml

@@ -98,6 +98,9 @@ spec:
                   - type
                   type: object
                 type: array
+              enableMultiMaster:
+                description: whether the DB instance is using multi-master routing
+                type: boolean
               hash:
                 additionalProperties:
                   type: string

controllers/galera_controller.go

@@ -509,16 +509,23 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 		// service get stuck.
 		controllerutil.AddFinalizer(service, helper.GetFinalizer())
 
-		// NOTE(dciabrin) We deploy Galera as an A/P service (i.e. no multi-master writes)
-		// by setting labels in the service's label selectors.
-		// This label is dynamically set based on the status of the Galera cluster,
-		// so in this CreateOrPatch block we must reuse whatever is present in
-		// the existing service CR in case we're patching it.
-		activePod, present := service.Spec.Selector[mariadb.ActivePodSelectorKey]
-		service.Spec = pkgsvc.Spec
-		if present {
-			service.Spec.Selector[mariadb.ActivePodSelectorKey] = activePod
+		if !instance.Spec.EnableMultiMaster {
+			// NOTE(dciabrin) We deploy Galera as an A/P service (i.e. no multi-master writes)
+			// by setting labels in the service's label selectors.
+			// This label is dynamically set based on the status of the Galera cluster,
+			// so in this CreateOrPatch block we must reuse whatever is present in
+			// the existing service CR in case we're patching it.
+			activePod, present := service.Spec.Selector[mariadb.ActivePodSelectorKey]
+			service.Spec = pkgsvc.Spec
+
+			if present {
+				service.Spec.Selector[mariadb.ActivePodSelectorKey] = activePod
+			}
+		} else {
+			service.Spec = pkgsvc.Spec
+			delete(service.Spec.Selector, mariadb.ActivePodSelectorKey)
 		}
+
 		err := controllerutil.SetControllerReference(instance, service, r.Client.Scheme())
 		if err != nil {
 			return err

controllers/mariadbdatabase_controller.go

@@ -27,6 +27,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper"
@@ -145,7 +147,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 
 	// here we know that Galera exists so add a finalizer to ourselves and to the db CR. Before this point there is no reason to have a finalizer on ourselves as nothing to cleanup.
 	if instance.DeletionTimestamp.IsZero() || isNewInstance { // this condition can be removed if you wish as it is always true at this point otherwise we would returned earlier.
-		if controllerutil.AddFinalizer(dbGalera, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) {
+		if dbGalera.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(dbGalera, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) {
 			err := r.Update(ctx, dbGalera)
 			if err != nil {
 				return ctrl.Result{}, err
@@ -200,6 +202,9 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		return ctrl.Result{}, err
 	}
 
+	// DB instances setup for multi master
+	instance.Status.EnableMultiMaster = dbGalera.Spec.EnableMultiMaster
+
 	dbCreateHash := instance.Status.Hash[databasev1beta1.DbCreateHash]
 	dbCreateJob := job.NewJob(
 		jobDef,
@@ -245,8 +250,43 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 
 // SetupWithManager -
 func (r *MariaDBDatabaseReconciler) SetupWithManager(mgr ctrl.Manager) error {
+
+	updateStatusFn := func(ctx context.Context, o client.Object) []reconcile.Request {
+		log := GetLog(ctx, "MariaDBDatabase")
+
+		result := []reconcile.Request{}
+
+		mariaDBDatabases := &databasev1beta1.MariaDBDatabaseList{}
+
+		listOpts := []client.ListOption{
+			client.InNamespace(o.GetNamespace()),
+		}
+		if err := r.Client.List(ctx, mariaDBDatabases, listOpts...); err != nil {
+			log.Error(err, "Unable to retrieve MariaDBDatabase CRs %w")
+			return nil
+		}
+
+		for _, cr := range mariaDBDatabases.Items {
+
+			if o.GetName() == cr.GetLabels()["dbName"] {
+				name := client.ObjectKey{
+					Namespace: o.GetNamespace(),
+					Name:      cr.Name,
+				}
+				log.Info(fmt.Sprintf("Galera %s is used by MariaDBDatabase CR %s", o.GetName(), cr.Name))
+				result = append(result, reconcile.Request{NamespacedName: name})
+			}
+		}
+
+		if len(result) > 0 {
+			return result
+		}
+		return nil
+	}
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&databasev1beta1.MariaDBDatabase{}).
+		Watches(&databasev1beta1.Galera{}, handler.EnqueueRequestsFromMapFunc(updateStatusFn)).
 		Complete(r)
 }
 

tests/kuttl/tests/galera_multi_master/01-deploy_ap_galera.yaml

@@ -0,0 +1,19 @@
+apiVersion: mariadb.openstack.org/v1beta1
+kind: Galera
+metadata:
+  name: openstack
+spec:
+  secret: osp-secret
+  storageClass: local-storage
+  storageRequest: 500M
+  replicas: 3
+  enableMultiMaster: false
+---
+apiVersion: mariadb.openstack.org/v1beta1
+kind: MariaDBDatabase
+metadata:
+  name: mydatabase
+  labels:
+    dbName: openstack
+spec:
+  name: mydatabase

tests/kuttl/tests/galera_multi_master/02-assert.yaml

@@ -0,0 +1,131 @@
+#
+# Check for:
+#
+# - 1 MariaDB CR
+# - 1 Pod for MariaDB CR
+#
+
+apiVersion: mariadb.openstack.org/v1beta1
+kind: Galera
+metadata:
+  name: openstack
+spec:
+  replicas: 3
+  secret: osp-secret
+  storageRequest: 500M
+status:
+  bootstrapped: true
+  conditions:
+  - message: Setup complete
+    reason: Ready
+    status: "True"
+    type: Ready
+  - message: Deployment completed
+    reason: Ready
+    status: "True"
+    type: DeploymentReady
+  - message: Exposing service completed
+    reason: Ready
+    status: "True"
+    type: ExposeServiceReady
+  - message: Input data complete
+    reason: Ready
+    status: "True"
+    type: InputReady
+  - message: RoleBinding created
+    reason: Ready
+    status: "True"
+    type: RoleBindingReady
+  - message: Role created
+    reason: Ready
+    status: "True"
+    type: RoleReady
+  - message: ServiceAccount created
+    reason: Ready
+    status: "True"
+    type: ServiceAccountReady
+  - message: Service config create completed
+    reason: Ready
+    status: "True"
+    type: ServiceConfigReady
+  - message: Input data complete
+    reason: Ready
+    status: "True"
+    type: TLSInputReady
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: openstack-galera
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: galera
+      cr: galera-openstack
+      galera/name: openstack
+  serviceName: openstack-galera
+  template:
+    metadata:
+      labels:
+        app: galera
+        cr: galera-openstack
+        galera/name: openstack
+    spec:
+      containers:
+      - command:
+        - /usr/bin/dumb-init
+        - --
+        - /usr/local/bin/kolla_start
+        name: galera
+        ports:
+        - containerPort: 3306
+          name: mysql
+          protocol: TCP
+        - containerPort: 4567
+          name: galera
+          protocol: TCP
+      serviceAccount: galera-openstack
+      serviceAccountName: galera-openstack
+status:
+  availableReplicas: 3
+  readyReplicas: 3
+  replicas: 3
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: openstack-galera-0
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: openstack-galera-1
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: openstack-galera-2
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openstack-galera
+spec:
+  ports:
+  - name: mysql
+    port: 3306
+    protocol: TCP
+    targetPort: 3306
+  selector:
+    app: galera
+    cr: galera-openstack
+---
+apiVersion: mariadb.openstack.org/v1beta1
+kind: MariaDBDatabase
+metadata:
+  labels:
+    dbName: openstack
+  name: mydatabase
+spec:
+  name: mydatabase

tests/kuttl/tests/galera_multi_master/03-assert.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      # ensure kubernetes pod key present
+      oc get -n ${NAMESPACE} service openstack -o yaml | grep "statefulset.kubernetes.io/pod-name: openstack-galera-"

tests/kuttl/tests/galera_multi_master/04-deploy_mm_galera.yaml

@@ -0,0 +1,10 @@
+apiVersion: mariadb.openstack.org/v1beta1
+kind: Galera
+metadata:
+  name: openstack
+spec:
+  secret: osp-secret
+  storageClass: local-storage
+  storageRequest: 500M
+  replicas: 3
+  enableMultiMaster: true

tests/kuttl/tests/galera_multi_master/05-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      # ensure kubernetes pod key *not* present
+      ! oc get -n ${NAMESPACE} service openstack -o yaml | grep "statefulset.kubernetes.io/pod-name: openstack-galera-"

tests/kuttl/tests/galera_multi_master/06-assert.yaml

@@ -0,0 +1,20 @@
+apiVersion: mariadb.openstack.org/v1beta1
+kind: Galera
+metadata:
+  name: openstack
+spec:
+  replicas: 3
+  secret: osp-secret
+  storageRequest: 500M
+  enableMultiMaster: true
+---
+apiVersion: mariadb.openstack.org/v1beta1
+kind: MariaDBDatabase
+metadata:
+  labels:
+    dbName: openstack
+  name: mydatabase
+spec:
+  name: mydatabase
+status:
+  enableMultiMaster: true

tests/kuttl/tests/galera_multi_master/07-teardown.yaml

@@ -0,0 +1,13 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+  - apiVersion: mariadb.openstack.org/v1beta1
+    kind: Galera
+    name: openstack
+  - apiVersion: mariadb.openstack.org/v1beta1
+    kind: MariaDBDatabase
+    name: mydatabase
+commands:
+  - script: |
+      oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2
+      for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done