From a8943797e01d7ef97a69935d1a24edf9a11f0ae3 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 27 Aug 2024 13:16:03 -0400 Subject: [PATCH] add multi-master option to galera This change adds the new value enableMultiMaster to the Galera CR defaulting to false; when set to true, the "statefulset.kubernetes.io/pod-name" key is removed from the mariadb service spec, and when set to false it is restored. This has the effect of publishing the service either under one specific pod for active/passive use, or using any pod for any request which is multi-master use. the value of the attribute is also propigated out to the status within the MariaDBDatabase CR, which is then consumable by openstack services in a similar manner as the TLS settings. Openstack services will then want to set wsrep_sync_wait accordingly for AP or MM mode, which will be proposed in followup changes. Fixes: OSPRH-7406 --- api/bases/mariadb.openstack.org_galeras.yaml | 5 + ...ariadb.openstack.org_mariadbdatabases.yaml | 3 + api/v1beta1/galera_types.go | 3 + api/v1beta1/mariadbdatabase_types.go | 3 + .../bases/mariadb.openstack.org_galeras.yaml | 5 + ...ariadb.openstack.org_mariadbdatabases.yaml | 3 + controllers/galera_controller.go | 25 ++-- controllers/mariadbdatabase_controller.go | 42 +++++- .../01-deploy_ap_galera.yaml | 19 +++ .../tests/galera_multi_master/02-assert.yaml | 131 ++++++++++++++++++ .../tests/galera_multi_master/03-assert.yaml | 7 + .../04-deploy_mm_galera.yaml | 10 ++ .../tests/galera_multi_master/05-assert.yaml | 6 + .../tests/galera_multi_master/06-assert.yaml | 20 +++ .../galera_multi_master/07-teardown.yaml | 13 ++ 15 files changed, 285 insertions(+), 10 deletions(-) create mode 100644 tests/kuttl/tests/galera_multi_master/01-deploy_ap_galera.yaml create mode 100644 tests/kuttl/tests/galera_multi_master/02-assert.yaml create mode 100644 tests/kuttl/tests/galera_multi_master/03-assert.yaml create mode 100644 tests/kuttl/tests/galera_multi_master/04-deploy_mm_galera.yaml create mode 100644 tests/kuttl/tests/galera_multi_master/05-assert.yaml create mode 100644 tests/kuttl/tests/galera_multi_master/06-assert.yaml create mode 100644 tests/kuttl/tests/galera_multi_master/07-teardown.yaml diff --git a/api/bases/mariadb.openstack.org_galeras.yaml b/api/bases/mariadb.openstack.org_galeras.yaml index 709dd51c..18221fc2 100644 --- a/api/bases/mariadb.openstack.org_galeras.yaml +++ b/api/bases/mariadb.openstack.org_galeras.yaml @@ -64,6 +64,10 @@ spec: description: When TLS is configured, only allow connections to the DB over TLS type: boolean + enableMultiMaster: + default: false + description: Use multi-master service routing + type: boolean logToDisk: description: Log Galera pod's output to disk type: boolean @@ -104,6 +108,7 @@ spec: type: object required: - containerImage + - enableMultiMaster - replicas - secret - storageClass diff --git a/api/bases/mariadb.openstack.org_mariadbdatabases.yaml b/api/bases/mariadb.openstack.org_mariadbdatabases.yaml index e783f89c..faf56fe8 100644 --- a/api/bases/mariadb.openstack.org_mariadbdatabases.yaml +++ b/api/bases/mariadb.openstack.org_mariadbdatabases.yaml @@ -98,6 +98,9 @@ spec: - type type: object type: array + enableMultiMaster: + description: whether the DB instance is using multi-master routing + type: boolean hash: additionalProperties: type: string diff --git a/api/v1beta1/galera_types.go b/api/v1beta1/galera_types.go index 806003b7..9de833c7 100644 --- a/api/v1beta1/galera_types.go +++ b/api/v1beta1/galera_types.go @@ -89,6 +89,9 @@ type GaleraSpecCore struct { // +kubebuilder:validation:Optional // Log Galera pod's output to disk LogToDisk bool `json:"logToDisk"` + // +kubebuilder:default=false + // Use multi-master service routing + EnableMultiMaster bool `json:"enableMultiMaster"` } // GaleraAttributes holds startup information for a Galera host diff --git a/api/v1beta1/mariadbdatabase_types.go b/api/v1beta1/mariadbdatabase_types.go index fe9a8675..3fa8ad6a 100644 --- a/api/v1beta1/mariadbdatabase_types.go +++ b/api/v1beta1/mariadbdatabase_types.go @@ -55,6 +55,9 @@ type MariaDBDatabaseStatus struct { // Whether TLS is supported by the DB instance TLSSupport bool `json:"tlsSupport,omitempty"` + + // whether the DB instance is using multi-master routing + EnableMultiMaster bool `json:"enableMultiMaster,omitempty"` } //+kubebuilder:object:root=true diff --git a/config/crd/bases/mariadb.openstack.org_galeras.yaml b/config/crd/bases/mariadb.openstack.org_galeras.yaml index 709dd51c..18221fc2 100644 --- a/config/crd/bases/mariadb.openstack.org_galeras.yaml +++ b/config/crd/bases/mariadb.openstack.org_galeras.yaml @@ -64,6 +64,10 @@ spec: description: When TLS is configured, only allow connections to the DB over TLS type: boolean + enableMultiMaster: + default: false + description: Use multi-master service routing + type: boolean logToDisk: description: Log Galera pod's output to disk type: boolean @@ -104,6 +108,7 @@ spec: type: object required: - containerImage + - enableMultiMaster - replicas - secret - storageClass diff --git a/config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml b/config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml index e783f89c..faf56fe8 100644 --- a/config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml +++ b/config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml @@ -98,6 +98,9 @@ spec: - type type: object type: array + enableMultiMaster: + description: whether the DB instance is using multi-master routing + type: boolean hash: additionalProperties: type: string diff --git a/controllers/galera_controller.go b/controllers/galera_controller.go index 33dc4eac..77234293 100644 --- a/controllers/galera_controller.go +++ b/controllers/galera_controller.go @@ -509,16 +509,23 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // service get stuck. controllerutil.AddFinalizer(service, helper.GetFinalizer()) - // NOTE(dciabrin) We deploy Galera as an A/P service (i.e. no multi-master writes) - // by setting labels in the service's label selectors. - // This label is dynamically set based on the status of the Galera cluster, - // so in this CreateOrPatch block we must reuse whatever is present in - // the existing service CR in case we're patching it. - activePod, present := service.Spec.Selector[mariadb.ActivePodSelectorKey] - service.Spec = pkgsvc.Spec - if present { - service.Spec.Selector[mariadb.ActivePodSelectorKey] = activePod + if !instance.Spec.EnableMultiMaster { + // NOTE(dciabrin) We deploy Galera as an A/P service (i.e. no multi-master writes) + // by setting labels in the service's label selectors. + // This label is dynamically set based on the status of the Galera cluster, + // so in this CreateOrPatch block we must reuse whatever is present in + // the existing service CR in case we're patching it. + activePod, present := service.Spec.Selector[mariadb.ActivePodSelectorKey] + service.Spec = pkgsvc.Spec + + if present { + service.Spec.Selector[mariadb.ActivePodSelectorKey] = activePod + } + } else { + service.Spec = pkgsvc.Spec + delete(service.Spec.Selector, mariadb.ActivePodSelectorKey) } + err := controllerutil.SetControllerReference(instance, service, r.Client.Scheme()) if err != nil { return err diff --git a/controllers/mariadbdatabase_controller.go b/controllers/mariadbdatabase_controller.go index 52a6b3e2..4bff4074 100644 --- a/controllers/mariadbdatabase_controller.go +++ b/controllers/mariadbdatabase_controller.go @@ -27,6 +27,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" @@ -145,7 +147,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ // here we know that Galera exists so add a finalizer to ourselves and to the db CR. Before this point there is no reason to have a finalizer on ourselves as nothing to cleanup. if instance.DeletionTimestamp.IsZero() || isNewInstance { // this condition can be removed if you wish as it is always true at this point otherwise we would returned earlier. - if controllerutil.AddFinalizer(dbGalera, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) { + if dbGalera.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(dbGalera, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) { err := r.Update(ctx, dbGalera) if err != nil { return ctrl.Result{}, err @@ -200,6 +202,9 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } + // DB instances setup for multi master + instance.Status.EnableMultiMaster = dbGalera.Spec.EnableMultiMaster + dbCreateHash := instance.Status.Hash[databasev1beta1.DbCreateHash] dbCreateJob := job.NewJob( jobDef, @@ -245,8 +250,43 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ // SetupWithManager - func (r *MariaDBDatabaseReconciler) SetupWithManager(mgr ctrl.Manager) error { + + updateStatusFn := func(ctx context.Context, o client.Object) []reconcile.Request { + log := GetLog(ctx, "MariaDBDatabase") + + result := []reconcile.Request{} + + mariaDBDatabases := &databasev1beta1.MariaDBDatabaseList{} + + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(ctx, mariaDBDatabases, listOpts...); err != nil { + log.Error(err, "Unable to retrieve MariaDBDatabase CRs %w") + return nil + } + + for _, cr := range mariaDBDatabases.Items { + + if o.GetName() == cr.GetLabels()["dbName"] { + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + log.Info(fmt.Sprintf("Galera %s is used by MariaDBDatabase CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + + if len(result) > 0 { + return result + } + return nil + } + return ctrl.NewControllerManagedBy(mgr). For(&databasev1beta1.MariaDBDatabase{}). + Watches(&databasev1beta1.Galera{}, handler.EnqueueRequestsFromMapFunc(updateStatusFn)). Complete(r) } diff --git a/tests/kuttl/tests/galera_multi_master/01-deploy_ap_galera.yaml b/tests/kuttl/tests/galera_multi_master/01-deploy_ap_galera.yaml new file mode 100644 index 00000000..89852b21 --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/01-deploy_ap_galera.yaml @@ -0,0 +1,19 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + enableMultiMaster: false +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBDatabase +metadata: + name: mydatabase + labels: + dbName: openstack +spec: + name: mydatabase diff --git a/tests/kuttl/tests/galera_multi_master/02-assert.yaml b/tests/kuttl/tests/galera_multi_master/02-assert.yaml new file mode 100644 index 00000000..e7e9cd35 --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/02-assert.yaml @@ -0,0 +1,131 @@ +# +# Check for: +# +# - 1 MariaDB CR +# - 1 Pod for MariaDB CR +# + +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 3 + secret: osp-secret + storageRequest: 500M +status: + bootstrapped: true + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: Deployment completed + reason: Ready + status: "True" + type: DeploymentReady + - message: Exposing service completed + reason: Ready + status: "True" + type: ExposeServiceReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - message: Role created + reason: Ready + status: "True" + type: RoleReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady + - message: Service config create completed + reason: Ready + status: "True" + type: ServiceConfigReady + - message: Input data complete + reason: Ready + status: "True" + type: TLSInputReady +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openstack-galera +spec: + replicas: 3 + selector: + matchLabels: + app: galera + cr: galera-openstack + galera/name: openstack + serviceName: openstack-galera + template: + metadata: + labels: + app: galera + cr: galera-openstack + galera/name: openstack + spec: + containers: + - command: + - /usr/bin/dumb-init + - -- + - /usr/local/bin/kolla_start + name: galera + ports: + - containerPort: 3306 + name: mysql + protocol: TCP + - containerPort: 4567 + name: galera + protocol: TCP + serviceAccount: galera-openstack + serviceAccountName: galera-openstack +status: + availableReplicas: 3 + readyReplicas: 3 + replicas: 3 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-2 +--- +apiVersion: v1 +kind: Service +metadata: + name: openstack-galera +spec: + ports: + - name: mysql + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: galera + cr: galera-openstack +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBDatabase +metadata: + labels: + dbName: openstack + name: mydatabase +spec: + name: mydatabase diff --git a/tests/kuttl/tests/galera_multi_master/03-assert.yaml b/tests/kuttl/tests/galera_multi_master/03-assert.yaml new file mode 100644 index 00000000..ffdbf861 --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/03-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure kubernetes pod key present + oc get -n ${NAMESPACE} service openstack -o yaml | grep "statefulset.kubernetes.io/pod-name: openstack-galera-" diff --git a/tests/kuttl/tests/galera_multi_master/04-deploy_mm_galera.yaml b/tests/kuttl/tests/galera_multi_master/04-deploy_mm_galera.yaml new file mode 100644 index 00000000..a9cae944 --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/04-deploy_mm_galera.yaml @@ -0,0 +1,10 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + enableMultiMaster: true diff --git a/tests/kuttl/tests/galera_multi_master/05-assert.yaml b/tests/kuttl/tests/galera_multi_master/05-assert.yaml new file mode 100644 index 00000000..289bf73e --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/05-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure kubernetes pod key *not* present + ! oc get -n ${NAMESPACE} service openstack -o yaml | grep "statefulset.kubernetes.io/pod-name: openstack-galera-" diff --git a/tests/kuttl/tests/galera_multi_master/06-assert.yaml b/tests/kuttl/tests/galera_multi_master/06-assert.yaml new file mode 100644 index 00000000..aa2ff8ff --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/06-assert.yaml @@ -0,0 +1,20 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 3 + secret: osp-secret + storageRequest: 500M + enableMultiMaster: true +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBDatabase +metadata: + labels: + dbName: openstack + name: mydatabase +spec: + name: mydatabase +status: + enableMultiMaster: true diff --git a/tests/kuttl/tests/galera_multi_master/07-teardown.yaml b/tests/kuttl/tests/galera_multi_master/07-teardown.yaml new file mode 100644 index 00000000..d595779d --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/07-teardown.yaml @@ -0,0 +1,13 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack + - apiVersion: mariadb.openstack.org/v1beta1 + kind: MariaDBDatabase + name: mydatabase +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done