diff --git a/controllers/nova_controller.go b/controllers/nova_controller.go index c7fbd3c15..d6adc5d22 100644 --- a/controllers/nova_controller.go +++ b/controllers/nova_controller.go @@ -596,16 +596,14 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul listOpts := []client.ListOption{ client.InNamespace(instance.Namespace), } - r.Client.List(ctx, novaCellList, listOpts...) + if err := r.Client.List(ctx, novaCellList, listOpts...); err != nil { + return ctrl.Result{}, err + } for _, cr := range novaCellList.Items { _, ok := instance.Spec.CellTemplates[cr.Spec.CellName] if !ok { - err = r.Client.Delete(ctx, &cr) - if err != nil { - return ctrl.Result{}, err - } - err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers(ctx, h, "nova-"+cr.Spec.CellName, cr.Spec.CellDatabaseAccount, instance.Namespace) + err := r.ensureCellDeleted(ctx, h, instance, cr.Spec.CellName, apiTransportURL, secret, apiDB) if err != nil { return ctrl.Result{}, err } @@ -617,6 +615,63 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul return ctrl.Result{}, nil } +func (r *NovaReconciler) ensureCellDeleted( + ctx context.Context, + h *helper.Helper, + instance *novav1.Nova, + cellName string, + apiTransportURL string, + secret corev1.Secret, + apiDB *mariadbv1.Database, +) error { + Log := r.GetLogger(ctx) + cell := &novav1.NovaCell{ + ObjectMeta: metav1.ObjectMeta{ + Name: getNovaCellCRName(instance.Name, cellName), + Namespace: instance.Namespace, + }, + } + + // If it is not created by us, we don't touch it + if !OwnedBy(cell, instance) { + Log.Info("CellName isn't defined in the nova, but there is a "+ + "Cell CR not owned by us. Not deleting it.", + "cell", cell) + return nil + } + err := r.Client.Delete(ctx, cell) + if err != nil && k8s_errors.IsNotFound(err) { + return err + } + + configHash, scriptName, configName, err := r.ensureNovaManageJobSecret(ctx, h, instance, + cell, secret, cell.Spec.APIDatabaseHostname, apiTransportURL, apiDB) + if err != nil { + return err + } + inputHash, err := util.HashOfInputHashes(configHash) + if err != nil { + return err + } + + labels := map[string]string{ + common.AppSelector: NovaLabelPrefix, + } + jobDef := nova.CellDeleteJob(instance, cell, configName, scriptName, inputHash, labels) + job := job.NewJob( + jobDef, cell.Name+"-cell-mapping", + instance.Spec.PreserveJobs, r.RequeueTimeout, + instance.Status.RegisteredCells[cell.Name]) + + _, err = job.DoJob(ctx, h) + if err != nil { + return err + } + + Log.Info("Cell isn't defined in the nova, so deleted cell", "cell", cell) + return nil +} + func (r *NovaReconciler) initStatus( instance *novav1.Nova, ) error { diff --git a/controllers/novacell_controller.go b/controllers/novacell_controller.go index 795b317b1..3d412118e 100644 --- a/controllers/novacell_controller.go +++ b/controllers/novacell_controller.go @@ -44,6 +44,8 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/labels" "github.com/openstack-k8s-operators/lib-common/modules/common/service" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" + "github.com/openstack-k8s-operators/nova-operator/pkg/novaapi" novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" ) @@ -312,6 +314,16 @@ func (r *NovaCellReconciler) reconcileDelete( } } + dbName, accountName := novaapi.ServiceName+"-"+instance.Spec.CellName, instance.Spec.CellDatabaseAccount + db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, h, dbName, accountName, instance.ObjectMeta.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + if !k8s_errors.IsNotFound(err) { + if err := db.DeleteFinalizer(ctx, h); err != nil { + return err + } + } Log.Info("Reconciled delete successfully") return nil } diff --git a/pkg/nova/celldelete.go b/pkg/nova/celldelete.go new file mode 100644 index 000000000..9ee45b727 --- /dev/null +++ b/pkg/nova/celldelete.go @@ -0,0 +1,86 @@ +package nova + +import ( + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" +) + +func CellDeleteJob( + instance *novav1.Nova, + cell *novav1.NovaCell, + configName string, + scriptName string, + inputHash string, + labels map[string]string, +) *batchv1.Job { + args := []string{"-c", KollaServiceCommand} + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["KOLLA_BOOTSTRAP"] = env.SetValue("true") + envVars["CELL_NAME"] = env.SetValue(cell.Spec.CellName) + + // This is stored in the Job so that if the input of the job changes + // then it results in a new job hash and therefore lib-common will re-run + // the job + envVars["INPUT_HASH"] = env.SetValue(inputHash) + + env := env.MergeEnvs([]corev1.EnvVar{}, envVars) + + jobName := instance.Name + "-" + cell.Spec.CellName + "-cell-delete" + + volumes := []corev1.Volume{ + GetConfigVolume(configName), + GetScriptVolume(scriptName), + } + volumeMounts := []corev1.VolumeMount{ + GetConfigVolumeMount(), + GetScriptVolumeMount(), + GetKollaConfigVolumeMount("cell-delete"), + } + + // add CA cert if defined + if instance.Spec.APIServiceTemplate.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.APIServiceTemplate.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.APIServiceTemplate.TLS.CreateVolumeMounts(nil)...) + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + Volumes: volumes, + Containers: []corev1.Container{ + { + Name: "nova-manage", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: cell.Spec.ConductorContainerImageURL, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(NovaUserID), + }, + Env: env, + VolumeMounts: volumeMounts, + }, + }, + }, + }, + }, + } + + return job +} diff --git a/templates/nova-manage/bin/delete_cell.sh b/templates/nova-manage/bin/delete_cell.sh new file mode 100755 index 000000000..0356324de --- /dev/null +++ b/templates/nova-manage/bin/delete_cell.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright 2023. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +export CELL_NAME=${CELL_NAME:?"Please specify a CELL_NAME variable."} + +# NOTE(gibi): nova-manage should be enhanced upstream to get rid of this +# uglyness +# Note the "|" around the CELL_NAME, that is needed as a single line from +# nova-manage cell_v2 cell_list can match to multiple cells if the cell name +# is part of the line, e.g. as the user name of the DB URL +cell_uuid=$(nova-manage cell_v2 list_cells | tr ' ' '|' | tr --squeeze-repeats '|' | grep -e "^|$CELL_NAME|" | cut -d '|' -f 3) + +if [ -z "${cell_uuid}" ]; then + nova-manage cell_v2 delete_cell --cell_uuid "${cell_uuid}" +fi diff --git a/templates/nova-manage/config/cell-delete-config.json b/templates/nova-manage/config/cell-delete-config.json new file mode 100644 index 000000000..8866cf24e --- /dev/null +++ b/templates/nova-manage/config/cell-delete-config.json @@ -0,0 +1,36 @@ +{ + "command": "/bin/ensure_cell_mapping.sh", + "config_files": [ + { + "source": "/var/lib/openstack/config/nova-blank.conf", + "dest": "/etc/nova/nova.conf", + "owner": "nova", + "perm": "0600" + }, + { + "source": "/var/lib/openstack/config/01-nova.conf", + "dest": "/etc/nova/nova.conf.d/01-nova.conf", + "owner": "nova", + "perm": "0600" + }, + { + "source": "/var/lib/openstack/config/02-nova-override.conf", + "dest": "/etc/nova/nova.conf.d/02-nova-override.conf", + "owner": "nova", + "perm": "0600", + "optional": true + }, + { + "source": "/var/lib/openstack/bin/delete_cell.sh", + "dest": "/bin/", + "owner": "nova", + "perm": "0700" + }, + { + "source": "/var/lib/openstack/config/my.cnf", + "dest": "/etc/my.cnf", + "owner": "nova", + "perm": "0644" + } + ] +} diff --git a/test/functional/nova_reconfiguration_test.go b/test/functional/nova_reconfiguration_test.go index 34f44e3bb..ce2cd3dc8 100644 --- a/test/functional/nova_reconfiguration_test.go +++ b/test/functional/nova_reconfiguration_test.go @@ -168,25 +168,23 @@ var _ = Describe("Nova reconfiguration", func() { CreateNovaWith3CellsAndEnsureReady(novaNames) }) - When("cell2 is deleted", func() { + When("cell1 is deleted", func() { It("cell cr is deleted", func() { Eventually(func(g Gomega) { nova := GetNova(novaNames.NovaName) - delete(nova.Spec.CellTemplates, "cell2") + delete(nova.Spec.CellTemplates, "cell1") g.Expect(k8sClient.Update(ctx, nova)).To(Succeed()) }, timeout, interval).Should(Succeed()) Eventually(func(g Gomega) { nova := GetNova(novaNames.NovaName) - g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell2.CellCRName.Name)) + g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell1.CellCRName.Name)) }, timeout, interval).Should(Succeed()) - Eventually(func(g Gomega) { - instance := &novav1.NovaCell{} - g.Expect(k8sClient.Get(ctx, cell2.CellCRName, instance)).ShouldNot(Succeed()) - }, timeout, interval).Should(Succeed()) + th.DeleteInstance(GetNovaCell(cell1.CellCRName)) + NovaCellNotExists(cell1.CellCRName) }) }) When("cell0 conductor replicas is set to 0", func() {