Add cell to clear cell from db

controllers/nova_controller.go

@@ -596,16 +596,14 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul
 	listOpts := []client.ListOption{
 		client.InNamespace(instance.Namespace),
 	}
-	r.Client.List(ctx, novaCellList, listOpts...)
+	if err := r.Client.List(ctx, novaCellList, listOpts...); err != nil {
+		return ctrl.Result{}, err
+	}
 
 	for _, cr := range novaCellList.Items {
 		_, ok := instance.Spec.CellTemplates[cr.Spec.CellName]
 		if !ok {
-			err = r.Client.Delete(ctx, &cr)
-			if err != nil {
-				return ctrl.Result{}, err
-			}
-			err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers(ctx, h, "nova-"+cr.Spec.CellName, cr.Spec.CellDatabaseAccount, instance.Namespace)
+			err := r.ensureCellDeleted(ctx, h, instance, cr.Spec.CellName, apiTransportURL, secret, apiDB)
 			if err != nil {
 				return ctrl.Result{}, err
 			}
@@ -617,6 +615,63 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul
 	return ctrl.Result{}, nil
 }
 
+func (r *NovaReconciler) ensureCellDeleted(
+	ctx context.Context,
+	h *helper.Helper,
+	instance *novav1.Nova,
+	cellName string,
+	apiTransportURL string,
+	secret corev1.Secret,
+	apiDB *mariadbv1.Database,
+) error {
+	Log := r.GetLogger(ctx)
+	cell := &novav1.NovaCell{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      getNovaCellCRName(instance.Name, cellName),
+			Namespace: instance.Namespace,
+		},
+	}
+
+	// If it is not created by us, we don't touch it
+	if !OwnedBy(cell, instance) {
+		Log.Info("CellName isn't defined in the nova, but there is a  "+
+			"Cell CR not owned by us. Not deleting it.",
+			"cell", cell)
+		return nil
+	}
+	err := r.Client.Delete(ctx, cell)
+	if err != nil && k8s_errors.IsNotFound(err) {
+		return err
+	}
+
+	configHash, scriptName, configName, err := r.ensureNovaManageJobSecret(ctx, h, instance,
+		cell, secret, cell.Spec.APIDatabaseHostname, apiTransportURL, apiDB)
+	if err != nil {
+		return err
+	}
+	inputHash, err := util.HashOfInputHashes(configHash)
+	if err != nil {
+		return err
+	}
+
+	labels := map[string]string{
+		common.AppSelector: NovaLabelPrefix,
+	}
+	jobDef := nova.CellDeleteJob(instance, cell, configName, scriptName, inputHash, labels)
+	job := job.NewJob(
+		jobDef, cell.Name+"-cell-mapping",
+		instance.Spec.PreserveJobs, r.RequeueTimeout,
+		instance.Status.RegisteredCells[cell.Name])
+
+	_, err = job.DoJob(ctx, h)
+	if err != nil {
+		return err
+	}
+
+	Log.Info("Cell isn't defined in the nova, so deleted cell", "cell", cell)
+	return nil
+}
+
 func (r *NovaReconciler) initStatus(
 	instance *novav1.Nova,
 ) error {

controllers/novacell_controller.go

@@ -44,6 +44,8 @@ import (
 	"github.com/openstack-k8s-operators/lib-common/modules/common/labels"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/service"
 	util "github.com/openstack-k8s-operators/lib-common/modules/common/util"
+	mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
+	"github.com/openstack-k8s-operators/nova-operator/pkg/novaapi"
 
 	novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1"
 )
@@ -312,6 +314,16 @@ func (r *NovaCellReconciler) reconcileDelete(
 		}
 	}
 
+	dbName, accountName := novaapi.ServiceName+"-"+instance.Spec.CellName, instance.Spec.CellDatabaseAccount
+	db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, h, dbName, accountName, instance.ObjectMeta.Namespace)
+	if err != nil && !k8s_errors.IsNotFound(err) {
+		return err
+	}
+	if !k8s_errors.IsNotFound(err) {
+		if err := db.DeleteFinalizer(ctx, h); err != nil {
+			return err
+		}
+	}
 	Log.Info("Reconciled delete successfully")
 	return nil
 }

pkg/nova/celldelete.go

@@ -0,0 +1,86 @@
+package nova
+
+import (
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
+
+	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
+	novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1"
+)
+
+func CellDeleteJob(
+	instance *novav1.Nova,
+	cell *novav1.NovaCell,
+	configName string,
+	scriptName string,
+	inputHash string,
+	labels map[string]string,
+) *batchv1.Job {
+	args := []string{"-c", KollaServiceCommand}
+
+	envVars := map[string]env.Setter{}
+	envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS")
+	envVars["KOLLA_BOOTSTRAP"] = env.SetValue("true")
+	envVars["CELL_NAME"] = env.SetValue(cell.Spec.CellName)
+
+	// This is stored in the Job so that if the input of the job changes
+	// then it results in a new job hash and therefore lib-common will re-run
+	// the job
+	envVars["INPUT_HASH"] = env.SetValue(inputHash)
+
+	env := env.MergeEnvs([]corev1.EnvVar{}, envVars)
+
+	jobName := instance.Name + "-" + cell.Spec.CellName + "-cell-delete"
+
+	volumes := []corev1.Volume{
+		GetConfigVolume(configName),
+		GetScriptVolume(scriptName),
+	}
+	volumeMounts := []corev1.VolumeMount{
+		GetConfigVolumeMount(),
+		GetScriptVolumeMount(),
+		GetKollaConfigVolumeMount("cell-delete"),
+	}
+
+	// add CA cert if defined
+	if instance.Spec.APIServiceTemplate.TLS.CaBundleSecretName != "" {
+		volumes = append(volumes, instance.Spec.APIServiceTemplate.TLS.CreateVolume())
+		volumeMounts = append(volumeMounts, instance.Spec.APIServiceTemplate.TLS.CreateVolumeMounts(nil)...)
+	}
+
+	job := &batchv1.Job{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      jobName,
+			Namespace: instance.Namespace,
+			Labels:    labels,
+		},
+		Spec: batchv1.JobSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					RestartPolicy:      corev1.RestartPolicyOnFailure,
+					ServiceAccountName: instance.RbacResourceName(),
+					Volumes:            volumes,
+					Containers: []corev1.Container{
+						{
+							Name: "nova-manage",
+							Command: []string{
+								"/bin/bash",
+							},
+							Args:  args,
+							Image: cell.Spec.ConductorContainerImageURL,
+							SecurityContext: &corev1.SecurityContext{
+								RunAsUser: ptr.To(NovaUserID),
+							},
+							Env:          env,
+							VolumeMounts: volumeMounts,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	return job
+}

templates/nova-manage/bin/delete_cell.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright 2023.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+export CELL_NAME=${CELL_NAME:?"Please specify a CELL_NAME variable."}
+
+# NOTE(gibi): nova-manage should be enhanced upstream to get rid of this
+# uglyness
+# Note the "|" around the CELL_NAME, that is needed as a single line from
+# nova-manage cell_v2 cell_list can match to multiple cells if the cell name
+# is part of the line, e.g. as the user name of the DB URL
+cell_uuid=$(nova-manage cell_v2 list_cells | tr ' ' '|' | tr --squeeze-repeats '|' | grep -e "^|$CELL_NAME|" | cut -d '|' -f 3)
+
+if [ -z "${cell_uuid}" ]; then
+    nova-manage cell_v2 delete_cell --cell_uuid "${cell_uuid}"
+fi

templates/nova-manage/config/cell-delete-config.json

@@ -0,0 +1,36 @@
+{
+    "command": "/bin/ensure_cell_mapping.sh",
+    "config_files": [
+        {
+            "source": "/var/lib/openstack/config/nova-blank.conf",
+            "dest": "/etc/nova/nova.conf",
+            "owner": "nova",
+            "perm": "0600"
+        },
+        {
+            "source": "/var/lib/openstack/config/01-nova.conf",
+            "dest": "/etc/nova/nova.conf.d/01-nova.conf",
+            "owner": "nova",
+            "perm": "0600"
+        },
+        {
+            "source": "/var/lib/openstack/config/02-nova-override.conf",
+            "dest": "/etc/nova/nova.conf.d/02-nova-override.conf",
+            "owner": "nova",
+            "perm": "0600",
+            "optional": true
+        },
+        {
+            "source": "/var/lib/openstack/bin/delete_cell.sh",
+            "dest": "/bin/",
+            "owner": "nova",
+            "perm": "0700"
+        },
+        {
+            "source": "/var/lib/openstack/config/my.cnf",
+            "dest": "/etc/my.cnf",
+            "owner": "nova",
+            "perm": "0644"
+        }
+    ]
+}

test/functional/nova_reconfiguration_test.go

@@ -168,25 +168,23 @@ var _ = Describe("Nova reconfiguration", func() {
 
 		CreateNovaWith3CellsAndEnsureReady(novaNames)
 	})
-	When("cell2 is deleted", func() {
+	When("cell1 is deleted", func() {
 		It("cell cr is deleted", func() {
 			Eventually(func(g Gomega) {
 				nova := GetNova(novaNames.NovaName)
 
-				delete(nova.Spec.CellTemplates, "cell2")
+				delete(nova.Spec.CellTemplates, "cell1")
 
 				g.Expect(k8sClient.Update(ctx, nova)).To(Succeed())
 			}, timeout, interval).Should(Succeed())
 
 			Eventually(func(g Gomega) {
 				nova := GetNova(novaNames.NovaName)
-				g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell2.CellCRName.Name))
+				g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell1.CellCRName.Name))
 			}, timeout, interval).Should(Succeed())
 
-			Eventually(func(g Gomega) {
-				instance := &novav1.NovaCell{}
-				g.Expect(k8sClient.Get(ctx, cell2.CellCRName, instance)).ShouldNot(Succeed())
-			}, timeout, interval).Should(Succeed())
+			th.DeleteInstance(GetNovaCell(cell1.CellCRName))
+			NovaCellNotExists(cell1.CellCRName)
 		})
 	})
 	When("cell0 conductor replicas is set to 0", func() {