diff --git a/api/bases/mariadb.openstack.org_galeras.yaml b/api/bases/mariadb.openstack.org_galeras.yaml index 709dd51c..18221fc2 100644 --- a/api/bases/mariadb.openstack.org_galeras.yaml +++ b/api/bases/mariadb.openstack.org_galeras.yaml @@ -64,6 +64,10 @@ spec: description: When TLS is configured, only allow connections to the DB over TLS type: boolean + enableMultiMaster: + default: false + description: Use multi-master service routing + type: boolean logToDisk: description: Log Galera pod's output to disk type: boolean @@ -104,6 +108,7 @@ spec: type: object required: - containerImage + - enableMultiMaster - replicas - secret - storageClass diff --git a/api/bases/mariadb.openstack.org_mariadbdatabases.yaml b/api/bases/mariadb.openstack.org_mariadbdatabases.yaml index e783f89c..faf56fe8 100644 --- a/api/bases/mariadb.openstack.org_mariadbdatabases.yaml +++ b/api/bases/mariadb.openstack.org_mariadbdatabases.yaml @@ -98,6 +98,9 @@ spec: - type type: object type: array + enableMultiMaster: + description: whether the DB instance is using multi-master routing + type: boolean hash: additionalProperties: type: string diff --git a/api/v1beta1/galera_types.go b/api/v1beta1/galera_types.go index 806003b7..9de833c7 100644 --- a/api/v1beta1/galera_types.go +++ b/api/v1beta1/galera_types.go @@ -89,6 +89,9 @@ type GaleraSpecCore struct { // +kubebuilder:validation:Optional // Log Galera pod's output to disk LogToDisk bool `json:"logToDisk"` + // +kubebuilder:default=false + // Use multi-master service routing + EnableMultiMaster bool `json:"enableMultiMaster"` } // GaleraAttributes holds startup information for a Galera host diff --git a/api/v1beta1/mariadbdatabase_types.go b/api/v1beta1/mariadbdatabase_types.go index fe9a8675..3fa8ad6a 100644 --- a/api/v1beta1/mariadbdatabase_types.go +++ b/api/v1beta1/mariadbdatabase_types.go @@ -55,6 +55,9 @@ type MariaDBDatabaseStatus struct { // Whether TLS is supported by the DB instance TLSSupport bool `json:"tlsSupport,omitempty"` + + // whether the DB instance is using multi-master routing + EnableMultiMaster bool `json:"enableMultiMaster,omitempty"` } //+kubebuilder:object:root=true diff --git a/config/crd/bases/mariadb.openstack.org_galeras.yaml b/config/crd/bases/mariadb.openstack.org_galeras.yaml index 709dd51c..18221fc2 100644 --- a/config/crd/bases/mariadb.openstack.org_galeras.yaml +++ b/config/crd/bases/mariadb.openstack.org_galeras.yaml @@ -64,6 +64,10 @@ spec: description: When TLS is configured, only allow connections to the DB over TLS type: boolean + enableMultiMaster: + default: false + description: Use multi-master service routing + type: boolean logToDisk: description: Log Galera pod's output to disk type: boolean @@ -104,6 +108,7 @@ spec: type: object required: - containerImage + - enableMultiMaster - replicas - secret - storageClass diff --git a/config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml b/config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml index e783f89c..faf56fe8 100644 --- a/config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml +++ b/config/crd/bases/mariadb.openstack.org_mariadbdatabases.yaml @@ -98,6 +98,9 @@ spec: - type type: object type: array + enableMultiMaster: + description: whether the DB instance is using multi-master routing + type: boolean hash: additionalProperties: type: string diff --git a/controllers/galera_controller.go b/controllers/galera_controller.go index 33dc4eac..77234293 100644 --- a/controllers/galera_controller.go +++ b/controllers/galera_controller.go @@ -509,16 +509,23 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // service get stuck. controllerutil.AddFinalizer(service, helper.GetFinalizer()) - // NOTE(dciabrin) We deploy Galera as an A/P service (i.e. no multi-master writes) - // by setting labels in the service's label selectors. - // This label is dynamically set based on the status of the Galera cluster, - // so in this CreateOrPatch block we must reuse whatever is present in - // the existing service CR in case we're patching it. - activePod, present := service.Spec.Selector[mariadb.ActivePodSelectorKey] - service.Spec = pkgsvc.Spec - if present { - service.Spec.Selector[mariadb.ActivePodSelectorKey] = activePod + if !instance.Spec.EnableMultiMaster { + // NOTE(dciabrin) We deploy Galera as an A/P service (i.e. no multi-master writes) + // by setting labels in the service's label selectors. + // This label is dynamically set based on the status of the Galera cluster, + // so in this CreateOrPatch block we must reuse whatever is present in + // the existing service CR in case we're patching it. + activePod, present := service.Spec.Selector[mariadb.ActivePodSelectorKey] + service.Spec = pkgsvc.Spec + + if present { + service.Spec.Selector[mariadb.ActivePodSelectorKey] = activePod + } + } else { + service.Spec = pkgsvc.Spec + delete(service.Spec.Selector, mariadb.ActivePodSelectorKey) } + err := controllerutil.SetControllerReference(instance, service, r.Client.Scheme()) if err != nil { return err diff --git a/controllers/mariadbdatabase_controller.go b/controllers/mariadbdatabase_controller.go index 52a6b3e2..4bff4074 100644 --- a/controllers/mariadbdatabase_controller.go +++ b/controllers/mariadbdatabase_controller.go @@ -27,6 +27,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" @@ -145,7 +147,7 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ // here we know that Galera exists so add a finalizer to ourselves and to the db CR. Before this point there is no reason to have a finalizer on ourselves as nothing to cleanup. if instance.DeletionTimestamp.IsZero() || isNewInstance { // this condition can be removed if you wish as it is always true at this point otherwise we would returned earlier. - if controllerutil.AddFinalizer(dbGalera, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) { + if dbGalera.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(dbGalera, fmt.Sprintf("%s-%s", helper.GetFinalizer(), instance.Name)) { err := r.Update(ctx, dbGalera) if err != nil { return ctrl.Result{}, err @@ -200,6 +202,9 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } + // DB instances setup for multi master + instance.Status.EnableMultiMaster = dbGalera.Spec.EnableMultiMaster + dbCreateHash := instance.Status.Hash[databasev1beta1.DbCreateHash] dbCreateJob := job.NewJob( jobDef, @@ -245,8 +250,43 @@ func (r *MariaDBDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Requ // SetupWithManager - func (r *MariaDBDatabaseReconciler) SetupWithManager(mgr ctrl.Manager) error { + + updateStatusFn := func(ctx context.Context, o client.Object) []reconcile.Request { + log := GetLog(ctx, "MariaDBDatabase") + + result := []reconcile.Request{} + + mariaDBDatabases := &databasev1beta1.MariaDBDatabaseList{} + + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(ctx, mariaDBDatabases, listOpts...); err != nil { + log.Error(err, "Unable to retrieve MariaDBDatabase CRs %w") + return nil + } + + for _, cr := range mariaDBDatabases.Items { + + if o.GetName() == cr.GetLabels()["dbName"] { + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + log.Info(fmt.Sprintf("Galera %s is used by MariaDBDatabase CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + + if len(result) > 0 { + return result + } + return nil + } + return ctrl.NewControllerManagedBy(mgr). For(&databasev1beta1.MariaDBDatabase{}). + Watches(&databasev1beta1.Galera{}, handler.EnqueueRequestsFromMapFunc(updateStatusFn)). Complete(r) } diff --git a/tests/kuttl/tests/galera_cluster_restart/04-teardown.yaml b/tests/kuttl/tests/galera_cluster_restart/04-teardown.yaml new file mode 100644 index 00000000..d87f62c4 --- /dev/null +++ b/tests/kuttl/tests/galera_cluster_restart/04-teardown.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta +kind: TestStep +delete: +- apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done diff --git a/tests/kuttl/tests/galera_log_to_disk/03-teardown.yaml b/tests/kuttl/tests/galera_log_to_disk/03-teardown.yaml new file mode 100644 index 00000000..d87f62c4 --- /dev/null +++ b/tests/kuttl/tests/galera_log_to_disk/03-teardown.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta +kind: TestStep +delete: +- apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done diff --git a/tests/kuttl/tests/galera_multi_master/01-deploy_ap_galera.yaml b/tests/kuttl/tests/galera_multi_master/01-deploy_ap_galera.yaml new file mode 100644 index 00000000..89852b21 --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/01-deploy_ap_galera.yaml @@ -0,0 +1,19 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + enableMultiMaster: false +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBDatabase +metadata: + name: mydatabase + labels: + dbName: openstack +spec: + name: mydatabase diff --git a/tests/kuttl/tests/galera_multi_master/02-assert.yaml b/tests/kuttl/tests/galera_multi_master/02-assert.yaml new file mode 100644 index 00000000..e7e9cd35 --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/02-assert.yaml @@ -0,0 +1,131 @@ +# +# Check for: +# +# - 1 MariaDB CR +# - 1 Pod for MariaDB CR +# + +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 3 + secret: osp-secret + storageRequest: 500M +status: + bootstrapped: true + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: Deployment completed + reason: Ready + status: "True" + type: DeploymentReady + - message: Exposing service completed + reason: Ready + status: "True" + type: ExposeServiceReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - message: Role created + reason: Ready + status: "True" + type: RoleReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady + - message: Service config create completed + reason: Ready + status: "True" + type: ServiceConfigReady + - message: Input data complete + reason: Ready + status: "True" + type: TLSInputReady +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openstack-galera +spec: + replicas: 3 + selector: + matchLabels: + app: galera + cr: galera-openstack + galera/name: openstack + serviceName: openstack-galera + template: + metadata: + labels: + app: galera + cr: galera-openstack + galera/name: openstack + spec: + containers: + - command: + - /usr/bin/dumb-init + - -- + - /usr/local/bin/kolla_start + name: galera + ports: + - containerPort: 3306 + name: mysql + protocol: TCP + - containerPort: 4567 + name: galera + protocol: TCP + serviceAccount: galera-openstack + serviceAccountName: galera-openstack +status: + availableReplicas: 3 + readyReplicas: 3 + replicas: 3 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: openstack-galera-2 +--- +apiVersion: v1 +kind: Service +metadata: + name: openstack-galera +spec: + ports: + - name: mysql + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: galera + cr: galera-openstack +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBDatabase +metadata: + labels: + dbName: openstack + name: mydatabase +spec: + name: mydatabase diff --git a/tests/kuttl/tests/galera_multi_master/03-assert.yaml b/tests/kuttl/tests/galera_multi_master/03-assert.yaml new file mode 100644 index 00000000..ffdbf861 --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/03-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure kubernetes pod key present + oc get -n ${NAMESPACE} service openstack -o yaml | grep "statefulset.kubernetes.io/pod-name: openstack-galera-" diff --git a/tests/kuttl/tests/galera_multi_master/04-deploy_mm_galera.yaml b/tests/kuttl/tests/galera_multi_master/04-deploy_mm_galera.yaml new file mode 100644 index 00000000..a9cae944 --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/04-deploy_mm_galera.yaml @@ -0,0 +1,10 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + enableMultiMaster: true diff --git a/tests/kuttl/tests/galera_multi_master/05-assert.yaml b/tests/kuttl/tests/galera_multi_master/05-assert.yaml new file mode 100644 index 00000000..289bf73e --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/05-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + # ensure kubernetes pod key *not* present + ! oc get -n ${NAMESPACE} service openstack -o yaml | grep "statefulset.kubernetes.io/pod-name: openstack-galera-" diff --git a/tests/kuttl/tests/galera_multi_master/06-assert.yaml b/tests/kuttl/tests/galera_multi_master/06-assert.yaml new file mode 100644 index 00000000..aa2ff8ff --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/06-assert.yaml @@ -0,0 +1,20 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + replicas: 3 + secret: osp-secret + storageRequest: 500M + enableMultiMaster: true +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBDatabase +metadata: + labels: + dbName: openstack + name: mydatabase +spec: + name: mydatabase +status: + enableMultiMaster: true diff --git a/tests/kuttl/tests/galera_multi_master/07-teardown.yaml b/tests/kuttl/tests/galera_multi_master/07-teardown.yaml new file mode 100644 index 00000000..d595779d --- /dev/null +++ b/tests/kuttl/tests/galera_multi_master/07-teardown.yaml @@ -0,0 +1,13 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: mariadb.openstack.org/v1beta1 + kind: Galera + name: openstack + - apiVersion: mariadb.openstack.org/v1beta1 + kind: MariaDBDatabase + name: mydatabase +commands: + - script: | + oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2 + for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done