From c3082cda313cdb734c1c043b629e9e18fa5be3fa Mon Sep 17 00:00:00 2001 From: Kamil Sambor Date: Mon, 8 Jul 2024 13:32:49 +0200 Subject: [PATCH 1/9] Add remove cell logic --- controllers/nova_controller.go | 22 +++++++ controllers/novacell_controller.go | 62 ++++++++++++++++++++ test/functional/nova_reconfiguration_test.go | 21 +++++++ 3 files changed, 105 insertions(+) diff --git a/controllers/nova_controller.go b/controllers/nova_controller.go index dd0844f79..c7fbd3c15 100644 --- a/controllers/nova_controller.go +++ b/controllers/nova_controller.go @@ -591,6 +591,28 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul } } + // We need to check and delete cells + novaCellList := &novav1.NovaCellList{} + listOpts := []client.ListOption{ + client.InNamespace(instance.Namespace), + } + r.Client.List(ctx, novaCellList, listOpts...) + + for _, cr := range novaCellList.Items { + _, ok := instance.Spec.CellTemplates[cr.Spec.CellName] + if !ok { + err = r.Client.Delete(ctx, &cr) + if err != nil { + return ctrl.Result{}, err + } + err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers(ctx, h, "nova-"+cr.Spec.CellName, cr.Spec.CellDatabaseAccount, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + delete(instance.Status.RegisteredCells, cr.Name) + } + } + Log.Info("Successfully reconciled") return ctrl.Result{}, nil } diff --git a/controllers/novacell_controller.go b/controllers/novacell_controller.go index 829ef78dd..795b317b1 100644 --- a/controllers/novacell_controller.go +++ b/controllers/novacell_controller.go @@ -137,6 +137,10 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r } }() + if !instance.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.reconcileDelete(ctx, h, instance) + } + // For the compute config generation we need to read the input secrets _, result, secret, err := ensureSecret( ctx, @@ -283,6 +287,34 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r Log.Info("Successfully reconciled") return ctrl.Result{}, nil } +func (r *NovaCellReconciler) reconcileDelete( + ctx context.Context, + h *helper.Helper, + instance *novav1.NovaCell, +) error { + Log := r.GetLogger(ctx) + err := r.ensureNoVNCProxyDeleted(ctx, instance) + if err != nil { + return err + } + err = r.ensureMetadataDeleted(ctx, instance) + if err != nil { + return err + } + err = r.ensureConductorDeleted(ctx, instance) + if err != nil { + return err + } + for computeName := range instance.Status.NovaComputesStatus { + err = r.ensureNovaComputeDeleted(ctx, instance, computeName) + if err != nil { + return err + } + } + + Log.Info("Reconciled delete successfully") + return nil +} func (r *NovaCellReconciler) initStatus( instance *novav1.NovaCell, @@ -399,6 +431,36 @@ func (r *NovaCellReconciler) ensureConductor( return ctrl.Result{}, nil } +func (r *NovaCellReconciler) ensureConductorDeleted( + ctx context.Context, + instance *novav1.NovaCell, +) error { + Log := r.GetLogger(ctx) + conductorName := types.NamespacedName{ + Name: instance.Name + "-conductor", + Namespace: instance.GetNamespace(), + } + + conductor := &novav1.NovaConductor{} + err := r.Client.Get(ctx, conductorName, conductor) + if k8s_errors.IsNotFound(err) { + // Nothing to do as it does not exists + return nil + } + if err != nil { + return err + } + + err = r.Client.Delete(ctx, conductor) + if err != nil && k8s_errors.IsNotFound(err) { + return nil + } + Log.Info("Cell is deleted, so cell Conductoris deleted", + "Conductor", conductor) + + return nil +} + func getNoVNCProxyName(instance *novav1.NovaCell) types.NamespacedName { return types.NamespacedName{Namespace: instance.Namespace, Name: instance.Name + "-novncproxy"} } diff --git a/test/functional/nova_reconfiguration_test.go b/test/functional/nova_reconfiguration_test.go index 89db43011..34f44e3bb 100644 --- a/test/functional/nova_reconfiguration_test.go +++ b/test/functional/nova_reconfiguration_test.go @@ -168,6 +168,27 @@ var _ = Describe("Nova reconfiguration", func() { CreateNovaWith3CellsAndEnsureReady(novaNames) }) + When("cell2 is deleted", func() { + It("cell cr is deleted", func() { + Eventually(func(g Gomega) { + nova := GetNova(novaNames.NovaName) + + delete(nova.Spec.CellTemplates, "cell2") + + g.Expect(k8sClient.Update(ctx, nova)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + nova := GetNova(novaNames.NovaName) + g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell2.CellCRName.Name)) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + instance := &novav1.NovaCell{} + g.Expect(k8sClient.Get(ctx, cell2.CellCRName, instance)).ShouldNot(Succeed()) + }, timeout, interval).Should(Succeed()) + }) + }) When("cell0 conductor replicas is set to 0", func() { It("sets the deployment replicas to 0", func() { cell0DeploymentName := cell0.ConductorStatefulSetName From d829bf9a5aaf4025042b7722e58e0667ddd99f0e Mon Sep 17 00:00:00 2001 From: Kamil Sambor Date: Mon, 19 Aug 2024 15:18:01 +0200 Subject: [PATCH 2/9] Add job to clear cell from db --- controllers/nova_controller.go | 114 +++++++++++++++++- controllers/novacell_controller.go | 12 ++ controllers/novaconductor_controller.go | 17 +++ pkg/nova/celldelete.go | 86 +++++++++++++ templates/nova-manage/bin/delete_cell.sh | 27 +++++ .../config/cell-delete-config.json | 36 ++++++ test/functional/base_test.go | 13 ++ test/functional/nova_reconfiguration_test.go | 22 +++- 8 files changed, 317 insertions(+), 10 deletions(-) create mode 100644 pkg/nova/celldelete.go create mode 100755 templates/nova-manage/bin/delete_cell.sh create mode 100644 templates/nova-manage/config/cell-delete-config.json diff --git a/controllers/nova_controller.go b/controllers/nova_controller.go index c7fbd3c15..f58fab6b1 100644 --- a/controllers/nova_controller.go +++ b/controllers/nova_controller.go @@ -596,20 +596,35 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul listOpts := []client.ListOption{ client.InNamespace(instance.Namespace), } - r.Client.List(ctx, novaCellList, listOpts...) + if err := r.Client.List(ctx, novaCellList, listOpts...); err != nil { + return ctrl.Result{}, err + } for _, cr := range novaCellList.Items { + cellDbs := [][]string{} _, ok := instance.Spec.CellTemplates[cr.Spec.CellName] if !ok { - err = r.Client.Delete(ctx, &cr) + err := r.ensureCellDeleted(ctx, h, instance, + cr.Spec.CellName, apiTransportURL, + secret, apiDB, cellDBs[novav1.Cell0Name].Database.GetDatabaseHostname(), cells[novav1.Cell0Name]) if err != nil { return ctrl.Result{}, err } - err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers(ctx, h, "nova-"+cr.Spec.CellName, cr.Spec.CellDatabaseAccount, instance.Namespace) - if err != nil { + cellDbs = append(cellDbs, []string{novaapi.ServiceName + "-" + cr.Spec.CellName, instance.Spec.CellTemplates[cr.Spec.CellName].CellDatabaseAccount}) + delete(instance.Status.RegisteredCells, cr.Name) + } + // iterate over novaDbs and remove finalizers + for _, novaDb := range cellDbs { + dbName, accountName := novaDb[0], novaDb[1] + db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, h, dbName, accountName, instance.ObjectMeta.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { return ctrl.Result{}, err } - delete(instance.Status.RegisteredCells, cr.Name) + if !k8s_errors.IsNotFound(err) { + if err := db.DeleteFinalizer(ctx, h); err != nil { + return ctrl.Result{}, err + } + } } } @@ -617,6 +632,95 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul return ctrl.Result{}, nil } +func (r *NovaReconciler) ensureCellDeleted( + ctx context.Context, + h *helper.Helper, + instance *novav1.Nova, + cellName string, + apiTransportURL string, + topLevelSecret corev1.Secret, + apiDB *mariadbv1.Database, + APIDatabaseHostname string, + cell0 *novav1.NovaCell, +) error { + Log := r.GetLogger(ctx) + cell := &novav1.NovaCell{} + fullCellName := types.NamespacedName{ + Name: getNovaCellCRName(instance.Name, cellName), + Namespace: instance.GetNamespace(), + } + + err := r.Client.Get(ctx, fullCellName, cell) + if k8s_errors.IsNotFound(err) { + // Nothing to do as it does not exists + return nil + } + if err != nil { + return err + } + // If it is not created by us, we don't touch it + if !OwnedBy(cell, instance) { + Log.Info("CellName isn't defined in the nova, but there is a "+ + "Cell CR not owned by us. Not deleting it.", + "cell", cell) + return nil + } + err = r.Client.Delete(ctx, cell) + if err != nil && k8s_errors.IsNotFound(err) { + return err + } + + configHash, scriptName, configName, err := r.ensureNovaManageJobSecret(ctx, h, instance, + cell0, topLevelSecret, APIDatabaseHostname, apiTransportURL, apiDB) + if err != nil { + return err + } + inputHash, err := util.HashOfInputHashes(configHash) + if err != nil { + return err + } + + labels := map[string]string{ + common.AppSelector: NovaLabelPrefix, + } + jobDef := nova.CellDeleteJob(instance, cell, configName, scriptName, inputHash, labels) + job := job.NewJob( + jobDef, cell.Name+"-cell-delete", + instance.Spec.PreserveJobs, r.RequeueTimeout, + instance.Status.RegisteredCells[cell.Name]) + + _, err = job.DoJob(ctx, h) + if err != nil { + return err + } + + // Delete secrets + dbSecret := fmt.Sprintf("%s-db-secret", cell.Name) + err = secret.DeleteSecretsWithName(ctx, h, dbSecret, instance.Namespace) + if err != nil { + return err + } + secretName := getNovaCellCRName(instance.Name, cellName) + err = secret.DeleteSecretsWithName(ctx, h, secretName, instance.Namespace) + if err != nil { + return err + } + + // Delete transportURL cr + transportURL := &rabbitmqv1.TransportURL{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name + "-" + cellName + "-transport", + Namespace: instance.Namespace, + }, + } + err = r.Client.Delete(ctx, transportURL) + if err != nil { + return err + } + Log.Info("Cell isn't defined in the nova, so deleted cell", "cell", cell) + return nil +} + func (r *NovaReconciler) initStatus( instance *novav1.Nova, ) error { diff --git a/controllers/novacell_controller.go b/controllers/novacell_controller.go index 795b317b1..3d412118e 100644 --- a/controllers/novacell_controller.go +++ b/controllers/novacell_controller.go @@ -44,6 +44,8 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/labels" "github.com/openstack-k8s-operators/lib-common/modules/common/service" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" + "github.com/openstack-k8s-operators/nova-operator/pkg/novaapi" novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" ) @@ -312,6 +314,16 @@ func (r *NovaCellReconciler) reconcileDelete( } } + dbName, accountName := novaapi.ServiceName+"-"+instance.Spec.CellName, instance.Spec.CellDatabaseAccount + db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, h, dbName, accountName, instance.ObjectMeta.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + if !k8s_errors.IsNotFound(err) { + if err := db.DeleteFinalizer(ctx, h); err != nil { + return err + } + } Log.Info("Reconciled delete successfully") return nil } diff --git a/controllers/novaconductor_controller.go b/controllers/novaconductor_controller.go index c8080377d..b1c6e6013 100644 --- a/controllers/novaconductor_controller.go +++ b/controllers/novaconductor_controller.go @@ -730,6 +730,23 @@ func (r *NovaConductorReconciler) reconcileDelete( Log.Info("Removed finalizer from ourselves") } + _, result, secret, err := ensureSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, + nil, + h.GetClient(), + &instance.Status.Conditions, + r.RequeueTimeout, + ) + if (err != nil || result != ctrl.Result{}) { + return err + } + + err = r.cleanServiceFromNovaDb(ctx, h, instance, secret, Log) + if err != nil { + return err + } + Log.Info("Reconciled delete successfully") return nil } diff --git a/pkg/nova/celldelete.go b/pkg/nova/celldelete.go new file mode 100644 index 000000000..9ee45b727 --- /dev/null +++ b/pkg/nova/celldelete.go @@ -0,0 +1,86 @@ +package nova + +import ( + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" +) + +func CellDeleteJob( + instance *novav1.Nova, + cell *novav1.NovaCell, + configName string, + scriptName string, + inputHash string, + labels map[string]string, +) *batchv1.Job { + args := []string{"-c", KollaServiceCommand} + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["KOLLA_BOOTSTRAP"] = env.SetValue("true") + envVars["CELL_NAME"] = env.SetValue(cell.Spec.CellName) + + // This is stored in the Job so that if the input of the job changes + // then it results in a new job hash and therefore lib-common will re-run + // the job + envVars["INPUT_HASH"] = env.SetValue(inputHash) + + env := env.MergeEnvs([]corev1.EnvVar{}, envVars) + + jobName := instance.Name + "-" + cell.Spec.CellName + "-cell-delete" + + volumes := []corev1.Volume{ + GetConfigVolume(configName), + GetScriptVolume(scriptName), + } + volumeMounts := []corev1.VolumeMount{ + GetConfigVolumeMount(), + GetScriptVolumeMount(), + GetKollaConfigVolumeMount("cell-delete"), + } + + // add CA cert if defined + if instance.Spec.APIServiceTemplate.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.APIServiceTemplate.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.APIServiceTemplate.TLS.CreateVolumeMounts(nil)...) + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + Volumes: volumes, + Containers: []corev1.Container{ + { + Name: "nova-manage", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: cell.Spec.ConductorContainerImageURL, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(NovaUserID), + }, + Env: env, + VolumeMounts: volumeMounts, + }, + }, + }, + }, + }, + } + + return job +} diff --git a/templates/nova-manage/bin/delete_cell.sh b/templates/nova-manage/bin/delete_cell.sh new file mode 100755 index 000000000..21ed94591 --- /dev/null +++ b/templates/nova-manage/bin/delete_cell.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2023. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +export CELL_NAME=${CELL_NAME:?"Please specify a CELL_NAME variable."} + +# NOTE(gibi): nova-manage should be enhanced upstream to get rid of this +# uglyness +# Note the "|" around the CELL_NAME, that is needed as a single line from +# nova-manage cell_v2 cell_list can match to multiple cells if the cell name +# is part of the line, e.g. as the user name of the DB URL +cell_uuid=$(nova-manage cell_v2 list_cells | tr ' ' '|' | tr --squeeze-repeats '|' | grep -e "^|$CELL_NAME|" | cut -d '|' -f 3) + +nova-manage cell_v2 delete_cell --cell_uuid "${cell_uuid}" diff --git a/templates/nova-manage/config/cell-delete-config.json b/templates/nova-manage/config/cell-delete-config.json new file mode 100644 index 000000000..955ea4ca2 --- /dev/null +++ b/templates/nova-manage/config/cell-delete-config.json @@ -0,0 +1,36 @@ +{ + "command": "/bin/delete_cell.sh", + "config_files": [ + { + "source": "/var/lib/openstack/config/nova-blank.conf", + "dest": "/etc/nova/nova.conf", + "owner": "nova", + "perm": "0600" + }, + { + "source": "/var/lib/openstack/config/01-nova.conf", + "dest": "/etc/nova/nova.conf.d/01-nova.conf", + "owner": "nova", + "perm": "0600" + }, + { + "source": "/var/lib/openstack/config/02-nova-override.conf", + "dest": "/etc/nova/nova.conf.d/02-nova-override.conf", + "owner": "nova", + "perm": "0600", + "optional": true + }, + { + "source": "/var/lib/openstack/bin/delete_cell.sh", + "dest": "/bin/", + "owner": "nova", + "perm": "0700" + }, + { + "source": "/var/lib/openstack/config/my.cnf", + "dest": "/etc/my.cnf", + "owner": "nova", + "perm": "0644" + } + ] +} diff --git a/test/functional/base_test.go b/test/functional/base_test.go index d5e493666..b39a87bd9 100644 --- a/test/functional/base_test.go +++ b/test/functional/base_test.go @@ -214,6 +214,14 @@ func NovaConductorConditionGetter(name types.NamespacedName) condition.Condition return instance.Status.Conditions } +func AssertConductorDoesNotExist(name types.NamespacedName) { + instance := &novav1.NovaConductor{} + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, name, instance) + g.Expect(k8s_errors.IsNotFound(err)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) +} + func CreateNovaMessageBusSecret(cell CellNames) *corev1.Secret { s := th.CreateSecret( types.NamespacedName{Namespace: cell.CellCRName.Namespace, Name: fmt.Sprintf("%s-secret", cell.TransportURLName.Name)}, @@ -350,6 +358,7 @@ type CellNames struct { ConductorStatefulSetName types.NamespacedName TransportURLName types.NamespacedName CellMappingJobName types.NamespacedName + CellDeleteJobName types.NamespacedName MetadataName types.NamespacedName MetadataStatefulSetName types.NamespacedName MetadataConfigDataName types.NamespacedName @@ -418,6 +427,10 @@ func GetCellNames(novaName types.NamespacedName, cell string) CellNames { Namespace: novaName.Namespace, Name: cellName.Name + "-cell-mapping", }, + CellDeleteJobName: types.NamespacedName{ + Namespace: novaName.Namespace, + Name: cellName.Name + "-cell-delete", + }, ConductorConfigDataName: types.NamespacedName{ Namespace: novaName.Namespace, Name: cellConductor.Name + "-config-data", diff --git a/test/functional/nova_reconfiguration_test.go b/test/functional/nova_reconfiguration_test.go index 34f44e3bb..11c5b44bf 100644 --- a/test/functional/nova_reconfiguration_test.go +++ b/test/functional/nova_reconfiguration_test.go @@ -22,11 +22,13 @@ import ( "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports . "github.com/onsi/gomega" //revive:disable:dot-imports + k8s_errors "k8s.io/apimachinery/pkg/api/errors" //revive:disable-next-line:dot-imports . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -168,24 +170,34 @@ var _ = Describe("Nova reconfiguration", func() { CreateNovaWith3CellsAndEnsureReady(novaNames) }) - When("cell2 is deleted", func() { + When("cell1 is deleted", func() { It("cell cr is deleted", func() { Eventually(func(g Gomega) { nova := GetNova(novaNames.NovaName) - delete(nova.Spec.CellTemplates, "cell2") + delete(nova.Spec.CellTemplates, "cell1") g.Expect(k8sClient.Update(ctx, nova)).To(Succeed()) }, timeout, interval).Should(Succeed()) Eventually(func(g Gomega) { nova := GetNova(novaNames.NovaName) - g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell2.CellCRName.Name)) + g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell1.CellCRName.Name)) }, timeout, interval).Should(Succeed()) + NovaCellNotExists(cell1.CellCRName) Eventually(func(g Gomega) { - instance := &novav1.NovaCell{} - g.Expect(k8sClient.Get(ctx, cell2.CellCRName, instance)).ShouldNot(Succeed()) + mappingJob := th.GetJob(cell1.CellDeleteJobName) + newJobInputHash := GetEnvVarValue( + mappingJob.Spec.Template.Spec.Containers[0].Env, "INPUT_HASH", "") + g.Expect(newJobInputHash).NotTo(BeNil()) + }, timeout, interval).Should(Succeed()) + th.AssertSecretDoesNotExist(cell1.InternalCellSecretName) + + Eventually(func(g Gomega) { + instance := &rabbitmqv1.TransportURL{} + err := k8sClient.Get(ctx, cell1.TransportURLName, instance) + g.Expect(k8s_errors.IsNotFound(err)).To(BeTrue()) }, timeout, interval).Should(Succeed()) }) }) From b8967534fb25d75a2ea6a3e13e77e722ce28abc5 Mon Sep 17 00:00:00 2001 From: Kamil Sambor Date: Fri, 20 Sep 2024 10:42:52 +0200 Subject: [PATCH 3/9] Add kuttle test for cell deletion --- .zuul.yaml | 49 ++ .../ansible.cfg | 2 + .../inventory.ini | 9 + .../playbooks/deploy-deps.yaml | 18 + .../playbooks/run-kuttl.yaml | 26 + .../common/cleanup-nova.yaml | 7 + .../test-suites/cell-delete-tests/config.yaml | 14 + .../deps/OpenStackControlPlane.yaml | 21 + .../cell-delete-tests/deps/infra.yaml | 32 + .../cell-delete-tests/deps/keystone.yaml | 9 + .../cell-delete-tests/deps/kustomization.yaml | 43 ++ .../cell-delete-tests/deps/namespace.yaml | 4 + .../cell-delete-tests/deps/nova.yaml | 7 + .../cell-delete-tests/deps/placement.yaml | 9 + .../tests/00-cleanup-nova.yaml | 1 + .../cell-delete-tests/tests/01-assert.yaml | 623 ++++++++++++++++++ .../cell-delete-tests/tests/01-deploy.yaml | 6 + .../cell-delete-tests/tests/02-assert.yaml | 125 ++++ .../tests/02-delete-cell-nova.yaml | 5 + .../tests/03-cleanup-nova.yaml | 1 + 20 files changed, 1011 insertions(+) create mode 100644 ci/nova-operator-kuttl-cell-delete/ansible.cfg create mode 100644 ci/nova-operator-kuttl-cell-delete/inventory.ini create mode 100644 ci/nova-operator-kuttl-cell-delete/playbooks/deploy-deps.yaml create mode 100644 ci/nova-operator-kuttl-cell-delete/playbooks/run-kuttl.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/common/cleanup-nova.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/config.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/OpenStackControlPlane.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/infra.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/keystone.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/kustomization.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/namespace.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/nova.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/placement.yaml create mode 120000 test/kuttl/test-suites/cell-delete-tests/tests/00-cleanup-nova.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/tests/01-assert.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/tests/01-deploy.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/tests/02-assert.yaml create mode 100644 test/kuttl/test-suites/cell-delete-tests/tests/02-delete-cell-nova.yaml create mode 120000 test/kuttl/test-suites/cell-delete-tests/tests/03-cleanup-nova.yaml diff --git a/.zuul.yaml b/.zuul.yaml index a9e3193f2..823c45857 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -72,6 +72,54 @@ ip: 172.18.0.5 tenant: ip: 172.19.0.5 +- job: + name: nova-operator-kuttl-cell-delete + parent: nova-operator-base + dependencies: ["openstack-meta-content-provider"] + roles: + - zuul: github.com/openstack-k8s-operators/ci-framework + description: | + This job deploy a basic "Compute Starter Kit" topology + https://www.openstack.org/software/sample-configs#compute-starter-kit + that is the minium set of openstack services required to boot a vm. + pre-run: + - ci/nova-operator-kuttl-cell-delete/playbooks/deploy-deps.yaml + run: + - ci/nova-operator-kuttl-cell-delete/playbooks/run-kuttl.yaml + nodeset: centos-9-medium-crc-extracted-2-39-0-3xl + vars: + collection_namespace_override: "nova-kuttl-cell-delete-tests" + zuul_log_collection: true + extra-vars: + crc_ci_bootstrap_networking: + networks: + default: + range: 192.168.122.0/24 + mtu: 1500 + internal-api: + vlan: 20 + range: 172.17.0.0/24 + storage: + vlan: 21 + range: 172.18.0.0/24 + tenant: + vlan: 22 + range: 172.19.0.0/24 + instances: + controller: + networks: + default: + ip: 192.168.122.11 + crc: + networks: + default: + ip: 192.168.122.10 + internal-api: + ip: 172.17.0.5 + storage: + ip: 172.18.0.5 + tenant: + ip: 172.19.0.5 - job: name: nova-operator-tempest-multinode parent: podified-multinode-edpm-deployment-crc-3comp @@ -211,6 +259,7 @@ - nova-operator-kuttl - nova-operator-tempest-multinode - nova-operator-tempest-multinode-ceph + - nova-operator-kuttl-cell-delete - pragma: implied-branch-matchers: True diff --git a/ci/nova-operator-kuttl-cell-delete/ansible.cfg b/ci/nova-operator-kuttl-cell-delete/ansible.cfg new file mode 100644 index 000000000..765ca2213 --- /dev/null +++ b/ci/nova-operator-kuttl-cell-delete/ansible.cfg @@ -0,0 +1,2 @@ +[defaults] +roles_path = /usr/share/ansible/roles:/etc/ansible/roles:../../../ci-framework/roles diff --git a/ci/nova-operator-kuttl-cell-delete/inventory.ini b/ci/nova-operator-kuttl-cell-delete/inventory.ini new file mode 100644 index 000000000..390656c8f --- /dev/null +++ b/ci/nova-operator-kuttl-cell-delete/inventory.ini @@ -0,0 +1,9 @@ +[openshift] +controller ansible_connection=local + +[openshift:vars] +local_nova_operator_basedir= "~/repos/nova-operator" +local_log_dir= "/tmp/nova-operator-kuttl/logs" + +[all:vars] +ansible_user_dir="~/" diff --git a/ci/nova-operator-kuttl-cell-delete/playbooks/deploy-deps.yaml b/ci/nova-operator-kuttl-cell-delete/playbooks/deploy-deps.yaml new file mode 100644 index 000000000..e7ad00d1f --- /dev/null +++ b/ci/nova-operator-kuttl-cell-delete/playbooks/deploy-deps.yaml @@ -0,0 +1,18 @@ +--- +- hosts: controller + vars: + nova_operator_basedir: "{{local_nova_operator_basedir | default(ansible_user_dir + '/src/github.com/openstack-k8s-operators/nova-operator')}}" + tasks: + - name: install kuttl test_suite dependencies + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path | default(ansible_env.PATH) }}" + KUTTL_SUITE: "cell-delete-tests" + ansible.builtin.command: + cmd: make kuttl-test-prep + chdir: "{{nova_operator_basedir}}" + register: result + until: result.rc == 0 + retries: 3 + delay: 10 + changed_when: true diff --git a/ci/nova-operator-kuttl-cell-delete/playbooks/run-kuttl.yaml b/ci/nova-operator-kuttl-cell-delete/playbooks/run-kuttl.yaml new file mode 100644 index 000000000..b60878178 --- /dev/null +++ b/ci/nova-operator-kuttl-cell-delete/playbooks/run-kuttl.yaml @@ -0,0 +1,26 @@ +- hosts: controller + vars: + nova_operator_basedir: "{{local_nova_operator_basedir | default(ansible_user_dir + '/src/github.com/openstack-k8s-operators/nova-operator')}}" + kuttl_log_dir: "{{local_log_dir | default(ansible_user_dir + '/zuul-output/logs/controller')}}" + tasks: + - name: install kuttl test_suite dependencies + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path | default(ansible_env.PATH) }}" + KUTTL_SUITE: "cell-delete-tests" + ansible.builtin.command: + cmd: make kuttl-test-run + chdir: "{{nova_operator_basedir}}" + changed_when: true + register: kuttl_test_run + - name: Create log dir + ansible.builtin.file: + path: "{{kuttl_log_dir}}" + state: directory + mode: "0755" + recurse: yes + - name: save output to file + ansible.builtin.copy: + content: "{{kuttl_test_run.stdout}}" + dest: "{{kuttl_log_dir}}/kuttl-test-run.out" + when: kuttl_test_run.stdout is defined diff --git a/test/kuttl/test-suites/cell-delete-tests/common/cleanup-nova.yaml b/test/kuttl/test-suites/cell-delete-tests/common/cleanup-nova.yaml new file mode 100644 index 000000000..55caf71d6 --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/common/cleanup-nova.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: +- apiVersion: nova.openstack.org/v1beta1 + kind: Nova + name: nova-kuttl-cell + namespace: nova-kuttl-cell-delete-tests diff --git a/test/kuttl/test-suites/cell-delete-tests/config.yaml b/test/kuttl/test-suites/cell-delete-tests/config.yaml new file mode 100644 index 000000000..52464bafd --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/config.yaml @@ -0,0 +1,14 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +reportFormat: JSON +reportName: kuttl-default-results +namespace: nova-kuttl-cell-delete-tests +# we could set this lower, but the initial image pull can take a while +timeout: 300 +parallel: 1 +skipDelete: true +testDirs: + - test/kuttl/test-suites/cell-delete-tests/ +suppress: + - events +artifactsDir: test/kuttl/test-suites/cell-delete-tests/output diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/OpenStackControlPlane.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/OpenStackControlPlane.yaml new file mode 100644 index 000000000..f76d003d9 --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/deps/OpenStackControlPlane.yaml @@ -0,0 +1,21 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + storageClass: "crc-csi-hostpath-provisioner" + tls: + ingress: + enabled: true + podLevel: + enabled: false + ironic: + enabled: false + template: + ironicConductors: [] + manila: + enabled: false + template: + manilaShares: {} + horizon: + enabled: false diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/infra.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/infra.yaml new file mode 100644 index 000000000..1dd64e878 --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/deps/infra.yaml @@ -0,0 +1,32 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + mariadb: + enabled: false + galera: + enabled: true + templates: + openstack: + storageRequest: 500M + openstack-cell1: + storageRequest: 500M + rabbitmq: + templates: + rabbitmq: + replicas: 1 + rabbitmq-cell1: + replicas: 1 + memcached: + templates: + memcached: + replicas: 1 + ovn: + enabled: false + template: + ovnController: + external-ids: + ovn-encap-type: geneve + ovs: + enabled: false diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/keystone.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/keystone.yaml new file mode 100644 index 000000000..97f568e7b --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/deps/keystone.yaml @@ -0,0 +1,9 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + keystone: + template: + databaseInstance: openstack + secret: osp-secret diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/kustomization.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/kustomization.yaml new file mode 100644 index 000000000..0a0ec3454 --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/deps/kustomization.yaml @@ -0,0 +1,43 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: nova-kuttl-cell-delete-tests + +secretGenerator: +- literals: + - AdminPassword=password + - DbRootPassword=password + - DatabasePassword=password + - KeystoneDatabasePassword=password + - PlacementPassword=password + - PlacementDatabasePassword=password + - GlancePassword=password + - GlanceDatabasePassword=password + - NeutronPassword=password + - NeutronDatabasePassword=password + - NovaPassword=password + - NovaAPIDatabasePassword=password + - NovaCell0DatabasePassword=password + - NovaCell1DatabasePassword=password + - MetadataSecret=42 + name: osp-secret +generatorOptions: + disableNameSuffixHash: true + labels: + type: osp-secret + +resources: +- namespace.yaml +- OpenStackControlPlane.yaml + +patches: +- patch: |- + apiVersion: core.openstack.org/v1beta1 + kind: OpenStackControlPlane + metadata: + name: openstack + spec: + secret: osp-secret +- path: infra.yaml +- path: keystone.yaml +- path: placement.yaml +- path: nova.yaml diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/namespace.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/namespace.yaml new file mode 100644 index 000000000..68e0205cd --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/deps/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: nova-kuttl-cell-delete-tests diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/nova.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/nova.yaml new file mode 100644 index 000000000..289dfc1b2 --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/deps/nova.yaml @@ -0,0 +1,7 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + nova: + enabled: false diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/placement.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/placement.yaml new file mode 100644 index 000000000..ed54e8c42 --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/deps/placement.yaml @@ -0,0 +1,9 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + placement: + template: + databaseInstance: openstack + secret: osp-secret diff --git a/test/kuttl/test-suites/cell-delete-tests/tests/00-cleanup-nova.yaml b/test/kuttl/test-suites/cell-delete-tests/tests/00-cleanup-nova.yaml new file mode 120000 index 000000000..23d51de43 --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/tests/00-cleanup-nova.yaml @@ -0,0 +1 @@ +../common/cleanup-nova.yaml \ No newline at end of file diff --git a/test/kuttl/test-suites/cell-delete-tests/tests/01-assert.yaml b/test/kuttl/test-suites/cell-delete-tests/tests/01-assert.yaml new file mode 100644 index 000000000..54312a821 --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/tests/01-assert.yaml @@ -0,0 +1,623 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: Nova +metadata: + finalizers: + - openstack.org/nova + name: nova-kuttl-cell + namespace: nova-kuttl-cell-delete-tests +spec: + apiDatabaseInstance: openstack + apiDatabaseAccount: nova-api + apiMessageBusInstance: rabbitmq + apiServiceTemplate: + customServiceConfig: "" + replicas: 1 + resources: {} + cellTemplates: + cell0: + cellDatabaseInstance: openstack + cellDatabaseAccount: nova-cell0 + cellMessageBusInstance: rabbitmq + conductorServiceTemplate: + customServiceConfig: "" + replicas: 1 + resources: {} + hasAPIAccess: true + metadataServiceTemplate: + enabled: false + noVNCProxyServiceTemplate: + enabled: false + cell1: + cellDatabaseInstance: openstack-cell1 + cellDatabaseAccount: nova-cell1 + cellMessageBusInstance: rabbitmq-cell1 + conductorServiceTemplate: + customServiceConfig: "" + replicas: 1 + resources: {} + hasAPIAccess: true + metadataServiceTemplate: + enabled: false + noVNCProxyServiceTemplate: + enabled: true + customServiceConfig: "" + replicas: 1 + resources: {} + keystoneInstance: keystone + metadataServiceTemplate: + enabled: true + customServiceConfig: "" + replicas: 1 + resources: {} + passwordSelectors: + metadataSecret: MetadataSecret + service: NovaPassword + schedulerServiceTemplate: + customServiceConfig: "" + replicas: 1 + resources: {} + secret: osp-secret + serviceUser: nova +status: + apiServiceReadyCount: 1 + metadataServiceReadyCount: 1 + schedulerServiceReadyCount: 1 + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: Setup complete + reason: Ready + status: "True" + type: KeystoneServiceReady + - message: MariaDBAccount creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady + - message: ' Memcached instance has been provisioned' + reason: Ready + status: "True" + type: MemcachedReady + - message: DB create completed + reason: Ready + status: "True" + type: NovaAPIDBReady + - message: API message bus creation successfully + reason: Ready + status: "True" + type: NovaAPIMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaAPIReady + - message: All DBs created successfully + reason: Ready + status: "True" + type: NovaAllCellDBReady + - message: All NovaCells are ready + reason: Ready + status: "True" + type: NovaAllCellReady + - message: All message busses created successfully + reason: Ready + status: "True" + type: NovaAllCellsMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaMetadataReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaSchedulerReady + - message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - message: Role created + reason: Ready + status: "True" + type: RoleReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-api + name: nova-kuttl-cell-api-0 +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified + name: nova-kuttl-cell-api-api + ready: true + started: true + - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified + name: nova-kuttl-cell-api-log + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + service: nova-scheduler + name: nova-kuttl-cell-scheduler-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-cell-scheduler +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-scheduler:current-podified + name: nova-kuttl-cell-scheduler-scheduler + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-metadata + statefulset.kubernetes.io/pod-name: nova-kuttl-cell-metadata-0 + name: nova-kuttl-cell-metadata-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-cell-metadata +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified + name: nova-kuttl-cell-metadata-log + ready: true + started: true + - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified + name: nova-kuttl-cell-metadata-metadata + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + cell: cell0 + service: nova-conductor + statefulset.kubernetes.io/pod-name: nova-kuttl-cell-cell0-conductor-0 + name: nova-kuttl-cell-cell0-conductor-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-cell-cell0-conductor +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified + name: nova-kuttl-cell-cell0-conductor-conductor + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + cell: cell1 + service: nova-conductor + statefulset.kubernetes.io/pod-name: nova-kuttl-cell-cell1-conductor-0 + name: nova-kuttl-cell-cell1-conductor-0 + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-cell-cell1-conductor +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified + name: nova-kuttl-cell-cell1-conductor-conductor + ready: true + started: true +--- +apiVersion: v1 +kind: Service +metadata: + labels: + endpoint: internal + service: nova-api + name: nova-internal + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaAPI + name: nova-kuttl-cell-api +spec: + ports: + - name: nova-internal + selector: + service: nova-api +--- +apiVersion: v1 +kind: Service +metadata: + labels: + endpoint: public + service: nova-api + name: nova-public + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaAPI + name: nova-kuttl-cell-api +spec: + ports: + - name: nova-public + selector: + service: nova-api +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-api.openstack.org/name: nova-kuttl-cell-api + name: nova-kuttl-cell-api-config-data + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaAPI + name: nova-kuttl-cell-api +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-conductor.openstack.org/name: nova-kuttl-cell-cell0-conductor + name: nova-kuttl-cell-cell0-conductor-config-data + namespace: nova-kuttl-cell-delete-tests + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell-cell0-conductor +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-conductor.openstack.org/name: nova-kuttl-cell-cell0-conductor + name: nova-kuttl-cell-cell0-conductor-scripts + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell-cell0-conductor +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-conductor.openstack.org/name: nova-kuttl-cell-cell1-conductor + name: nova-kuttl-cell-cell1-conductor-config-data + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell-cell1-conductor +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-conductor.openstack.org/name: nova-kuttl-cell-cell1-conductor + name: nova-kuttl-cell-cell1-conductor-scripts + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell-cell1-conductor +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-metadata.openstack.org/name: nova-kuttl-cell-metadata + name: nova-kuttl-cell-metadata-config-data + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaMetadata + name: nova-kuttl-cell-metadata +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-scheduler.openstack.org/name: nova-kuttl-cell-scheduler + name: nova-kuttl-cell-scheduler-config-data + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaScheduler + name: nova-kuttl-cell-scheduler +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-cell.openstack.org/name: nova-kuttl-cell-cell1 + name: nova-kuttl-cell-cell1-compute-config + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaCell + name: nova-kuttl-cell-cell1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-cell-api + namespace: nova-kuttl-cell-delete-tests + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaAPI + name: nova-kuttl-cell-api +spec: + replicas: 1 + selector: + matchLabels: + service: nova-api + serviceName: "" + template: + metadata: + labels: + service: nova-api +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-cell-metadata + namespace: nova-kuttl-cell-delete-tests + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaMetadata + name: nova-kuttl-cell-metadata +spec: + replicas: 1 + selector: + matchLabels: + service: nova-metadata + template: + metadata: + labels: + service: nova-metadata +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-cell-scheduler + namespace: nova-kuttl-cell-delete-tests + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaScheduler + name: nova-kuttl-cell-scheduler +spec: + replicas: 1 + selector: + matchLabels: + service: nova-scheduler + template: + metadata: + labels: + service: nova-scheduler +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-cell-cell0-conductor + namespace: nova-kuttl-cell-delete-tests + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell-cell0-conductor +spec: + replicas: 1 + selector: + matchLabels: + cell: cell0 + service: nova-conductor + template: + metadata: + labels: + cell: cell0 + service: nova-conductor +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-cell-cell1-conductor + namespace: nova-kuttl-cell-delete-tests + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell-cell1-conductor +spec: + replicas: 1 + selector: + matchLabels: + cell: cell1 + service: nova-conductor + template: + metadata: + labels: + cell: cell1 + service: nova-conductor +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + labels: + service: nova-conductor + name: nova-kuttl-cell-cell0-db-purge + namespace: nova-kuttl-cell-delete-tests +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + completions: 1 + parallelism: 1 + template: + spec: + containers: + - args: + - -c + - /usr/local/bin/kolla_start + command: + - /bin/bash + env: + - name: ARCHIVE_AGE + value: "30" + - name: KOLLA_BOOTSTRAP + value: "true" + - name: KOLLA_CONFIG_STRATEGY + value: COPY_ALWAYS + - name: PURGE_AGE + value: "90" + image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified + name: nova-manage + securityContext: + runAsUser: 42436 + volumeMounts: + - mountPath: /var/lib/openstack/config + name: config-data + - mountPath: /var/lib/openstack/bin + name: scripts + - mountPath: /var/lib/kolla/config_files/config.json + name: config-data + subPath: nova-conductor-dbpurge-config.json + restartPolicy: OnFailure + serviceAccount: nova-nova-kuttl-cell + serviceAccountName: nova-nova-kuttl-cell + volumes: + - name: config-data + secret: + secretName: nova-kuttl-cell-cell0-conductor-config-data + - name: scripts + secret: + secretName: nova-kuttl-cell-cell0-conductor-scripts + schedule: 0 0 * * * + successfulJobsHistoryLimit: 3 + suspend: false +status: {} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + labels: + service: nova-conductor + name: nova-kuttl-cell-cell1-db-purge + namespace: nova-kuttl-cell-delete-tests +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + completions: 1 + parallelism: 1 + template: + spec: + containers: + - args: + - -c + - /usr/local/bin/kolla_start + command: + - /bin/bash + env: + - name: ARCHIVE_AGE + value: "30" + - name: KOLLA_BOOTSTRAP + value: "true" + - name: KOLLA_CONFIG_STRATEGY + value: COPY_ALWAYS + - name: PURGE_AGE + value: "90" + image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified + name: nova-manage + securityContext: + runAsUser: 42436 + volumeMounts: + - mountPath: /var/lib/openstack/config + name: config-data + - mountPath: /var/lib/openstack/bin + name: scripts + - mountPath: /var/lib/kolla/config_files/config.json + name: config-data + subPath: nova-conductor-dbpurge-config.json + restartPolicy: OnFailure + serviceAccount: nova-nova-kuttl-cell + serviceAccountName: nova-nova-kuttl-cell + volumes: + - name: config-data + secret: + secretName: nova-kuttl-cell-cell1-conductor-config-data + - name: scripts + secret: + secretName: nova-kuttl-cell-cell1-conductor-scripts + schedule: 0 0 * * * + successfulJobsHistoryLimit: 3 + suspend: false +status: {} diff --git a/test/kuttl/test-suites/cell-delete-tests/tests/01-deploy.yaml b/test/kuttl/test-suites/cell-delete-tests/tests/01-deploy.yaml new file mode 100644 index 000000000..8c1addcd5 --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/tests/01-deploy.yaml @@ -0,0 +1,6 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: Nova +metadata: + name: nova-kuttl-cell +spec: + secret: osp-secret diff --git a/test/kuttl/test-suites/cell-delete-tests/tests/02-assert.yaml b/test/kuttl/test-suites/cell-delete-tests/tests/02-assert.yaml new file mode 100644 index 000000000..a0151166e --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/tests/02-assert.yaml @@ -0,0 +1,125 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: Nova +metadata: + finalizers: + - openstack.org/nova + name: nova-kuttl-cell + namespace: nova-kuttl-cell-delete-tests +status: + apiServiceReadyCount: 1 + metadataServiceReadyCount: 1 + schedulerServiceReadyCount: 1 + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: Setup complete + reason: Ready + status: "True" + type: KeystoneServiceReady + - message: MariaDBAccount creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady + - message: ' Memcached instance has been provisioned' + reason: Ready + status: "True" + type: MemcachedReady + - message: DB create completed + reason: Ready + status: "True" + type: NovaAPIDBReady + - message: API message bus creation successfully + reason: Ready + status: "True" + type: NovaAPIMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaAPIReady + - message: All DBs created successfully + reason: Ready + status: "True" + type: NovaAllCellDBReady + - message: All NovaCells are ready + reason: Ready + status: "True" + type: NovaAllCellReady + - message: All message busses created successfully + reason: Ready + status: "True" + type: NovaAllCellsMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaMetadataReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaSchedulerReady + - message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - message: Role created + reason: Ready + status: "True" + type: RoleReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-api + name: nova-kuttl-cell-api-0 +status: + containerStatuses: + - name: nova-kuttl-cell-api-api + ready: true + started: true + - name: nova-kuttl-cell-api-log + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-metadata + statefulset.kubernetes.io/pod-name: nova-kuttl-cell-metadata-0 + name: nova-kuttl-cell-metadata-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-cell-metadata +status: + containerStatuses: + - name: nova-kuttl-cell-metadata-log + ready: true + started: true + - name: nova-kuttl-cell-metadata-metadata + ready: true + started: true + +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +namespaced: true +commands: + - command: echo oc get -n $NAMESPACE pod/nova-cell1-novncproxy-0 --ignore-not-found + - command: echo oc get -n $NAMESPACE pod/nova-cell1-conductor-0--ignore-not-found diff --git a/test/kuttl/test-suites/cell-delete-tests/tests/02-delete-cell-nova.yaml b/test/kuttl/test-suites/cell-delete-tests/tests/02-delete-cell-nova.yaml new file mode 100644 index 000000000..f1b5d904e --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/tests/02-delete-cell-nova.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc patch -n nova-kuttl-cell-delete-tests nova/nova-kuttl-cell --type='json' -p='[{"op": "remove", "path": "/spec/cellTemplates/cell1"}]' diff --git a/test/kuttl/test-suites/cell-delete-tests/tests/03-cleanup-nova.yaml b/test/kuttl/test-suites/cell-delete-tests/tests/03-cleanup-nova.yaml new file mode 120000 index 000000000..49b58eee9 --- /dev/null +++ b/test/kuttl/test-suites/cell-delete-tests/tests/03-cleanup-nova.yaml @@ -0,0 +1 @@ +00-cleanup-nova.yaml \ No newline at end of file From f4765e4315bc6f2cf732d419754fc4bc61c1538a Mon Sep 17 00:00:00 2001 From: Kamil Sambor Date: Mon, 23 Sep 2024 17:07:06 +0200 Subject: [PATCH 4/9] Correct cell deletion logic --- controllers/novacell_controller.go | 74 ------------------------- controllers/novaconductor_controller.go | 17 ------ 2 files changed, 91 deletions(-) diff --git a/controllers/novacell_controller.go b/controllers/novacell_controller.go index 3d412118e..829ef78dd 100644 --- a/controllers/novacell_controller.go +++ b/controllers/novacell_controller.go @@ -44,8 +44,6 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/labels" "github.com/openstack-k8s-operators/lib-common/modules/common/service" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" - mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" - "github.com/openstack-k8s-operators/nova-operator/pkg/novaapi" novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" ) @@ -139,10 +137,6 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r } }() - if !instance.DeletionTimestamp.IsZero() { - return ctrl.Result{}, r.reconcileDelete(ctx, h, instance) - } - // For the compute config generation we need to read the input secrets _, result, secret, err := ensureSecret( ctx, @@ -289,44 +283,6 @@ func (r *NovaCellReconciler) Reconcile(ctx context.Context, req ctrl.Request) (r Log.Info("Successfully reconciled") return ctrl.Result{}, nil } -func (r *NovaCellReconciler) reconcileDelete( - ctx context.Context, - h *helper.Helper, - instance *novav1.NovaCell, -) error { - Log := r.GetLogger(ctx) - err := r.ensureNoVNCProxyDeleted(ctx, instance) - if err != nil { - return err - } - err = r.ensureMetadataDeleted(ctx, instance) - if err != nil { - return err - } - err = r.ensureConductorDeleted(ctx, instance) - if err != nil { - return err - } - for computeName := range instance.Status.NovaComputesStatus { - err = r.ensureNovaComputeDeleted(ctx, instance, computeName) - if err != nil { - return err - } - } - - dbName, accountName := novaapi.ServiceName+"-"+instance.Spec.CellName, instance.Spec.CellDatabaseAccount - db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, h, dbName, accountName, instance.ObjectMeta.Namespace) - if err != nil && !k8s_errors.IsNotFound(err) { - return err - } - if !k8s_errors.IsNotFound(err) { - if err := db.DeleteFinalizer(ctx, h); err != nil { - return err - } - } - Log.Info("Reconciled delete successfully") - return nil -} func (r *NovaCellReconciler) initStatus( instance *novav1.NovaCell, @@ -443,36 +399,6 @@ func (r *NovaCellReconciler) ensureConductor( return ctrl.Result{}, nil } -func (r *NovaCellReconciler) ensureConductorDeleted( - ctx context.Context, - instance *novav1.NovaCell, -) error { - Log := r.GetLogger(ctx) - conductorName := types.NamespacedName{ - Name: instance.Name + "-conductor", - Namespace: instance.GetNamespace(), - } - - conductor := &novav1.NovaConductor{} - err := r.Client.Get(ctx, conductorName, conductor) - if k8s_errors.IsNotFound(err) { - // Nothing to do as it does not exists - return nil - } - if err != nil { - return err - } - - err = r.Client.Delete(ctx, conductor) - if err != nil && k8s_errors.IsNotFound(err) { - return nil - } - Log.Info("Cell is deleted, so cell Conductoris deleted", - "Conductor", conductor) - - return nil -} - func getNoVNCProxyName(instance *novav1.NovaCell) types.NamespacedName { return types.NamespacedName{Namespace: instance.Namespace, Name: instance.Name + "-novncproxy"} } diff --git a/controllers/novaconductor_controller.go b/controllers/novaconductor_controller.go index b1c6e6013..c8080377d 100644 --- a/controllers/novaconductor_controller.go +++ b/controllers/novaconductor_controller.go @@ -730,23 +730,6 @@ func (r *NovaConductorReconciler) reconcileDelete( Log.Info("Removed finalizer from ourselves") } - _, result, secret, err := ensureSecret( - ctx, - types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, - nil, - h.GetClient(), - &instance.Status.Conditions, - r.RequeueTimeout, - ) - if (err != nil || result != ctrl.Result{}) { - return err - } - - err = r.cleanServiceFromNovaDb(ctx, h, instance, secret, Log) - if err != nil { - return err - } - Log.Info("Reconciled delete successfully") return nil } From 1f6c1f67169af702234281353cc10eb309d46b1d Mon Sep 17 00:00:00 2001 From: Kamil Sambor Date: Wed, 25 Sep 2024 11:10:51 +0200 Subject: [PATCH 5/9] Move cell-delete test --- .../default/tests/00-cleanup-nova.yaml | 1 + .../test-suites/default/tests/01-assert.yaml | 623 ++++++++++++++++++ .../test-suites/default/tests/01-deploy.yaml | 6 + .../test-suites/default/tests/02-assert.yaml | 125 ++++ .../default/tests/02-delete-cell-nova.yaml | 5 + .../default/tests/03-cleanup-nova.yaml | 1 + 6 files changed, 761 insertions(+) create mode 120000 test/kuttl/test-suites/default/tests/00-cleanup-nova.yaml create mode 100644 test/kuttl/test-suites/default/tests/01-assert.yaml create mode 100644 test/kuttl/test-suites/default/tests/01-deploy.yaml create mode 100644 test/kuttl/test-suites/default/tests/02-assert.yaml create mode 100644 test/kuttl/test-suites/default/tests/02-delete-cell-nova.yaml create mode 120000 test/kuttl/test-suites/default/tests/03-cleanup-nova.yaml diff --git a/test/kuttl/test-suites/default/tests/00-cleanup-nova.yaml b/test/kuttl/test-suites/default/tests/00-cleanup-nova.yaml new file mode 120000 index 000000000..23d51de43 --- /dev/null +++ b/test/kuttl/test-suites/default/tests/00-cleanup-nova.yaml @@ -0,0 +1 @@ +../common/cleanup-nova.yaml \ No newline at end of file diff --git a/test/kuttl/test-suites/default/tests/01-assert.yaml b/test/kuttl/test-suites/default/tests/01-assert.yaml new file mode 100644 index 000000000..54a7bc6ca --- /dev/null +++ b/test/kuttl/test-suites/default/tests/01-assert.yaml @@ -0,0 +1,623 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: Nova +metadata: + finalizers: + - openstack.org/nova + name: nova-kuttl + namespace: nova-kuttl-default +spec: + apiDatabaseInstance: openstack + apiDatabaseAccount: nova-api + apiMessageBusInstance: rabbitmq + apiServiceTemplate: + customServiceConfig: "" + replicas: 1 + resources: {} + cellTemplates: + cell0: + cellDatabaseInstance: openstack + cellDatabaseAccount: nova-cell0 + cellMessageBusInstance: rabbitmq + conductorServiceTemplate: + customServiceConfig: "" + replicas: 1 + resources: {} + hasAPIAccess: true + metadataServiceTemplate: + enabled: false + noVNCProxyServiceTemplate: + enabled: false + cell1: + cellDatabaseInstance: openstack-cell1 + cellDatabaseAccount: nova-cell1 + cellMessageBusInstance: rabbitmq-cell1 + conductorServiceTemplate: + customServiceConfig: "" + replicas: 1 + resources: {} + hasAPIAccess: true + metadataServiceTemplate: + enabled: false + noVNCProxyServiceTemplate: + enabled: true + customServiceConfig: "" + replicas: 1 + resources: {} + keystoneInstance: keystone + metadataServiceTemplate: + enabled: true + customServiceConfig: "" + replicas: 1 + resources: {} + passwordSelectors: + metadataSecret: MetadataSecret + service: NovaPassword + schedulerServiceTemplate: + customServiceConfig: "" + replicas: 1 + resources: {} + secret: osp-secret + serviceUser: nova +status: + apiServiceReadyCount: 1 + metadataServiceReadyCount: 1 + schedulerServiceReadyCount: 1 + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: Setup complete + reason: Ready + status: "True" + type: KeystoneServiceReady + - message: MariaDBAccount creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady + - message: ' Memcached instance has been provisioned' + reason: Ready + status: "True" + type: MemcachedReady + - message: DB create completed + reason: Ready + status: "True" + type: NovaAPIDBReady + - message: API message bus creation successfully + reason: Ready + status: "True" + type: NovaAPIMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaAPIReady + - message: All DBs created successfully + reason: Ready + status: "True" + type: NovaAllCellDBReady + - message: All NovaCells are ready + reason: Ready + status: "True" + type: NovaAllCellReady + - message: All message busses created successfully + reason: Ready + status: "True" + type: NovaAllCellsMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaMetadataReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaSchedulerReady + - message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - message: Role created + reason: Ready + status: "True" + type: RoleReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-api + name: nova-kuttl-api-0 +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified + name: nova-kuttl-api-api + ready: true + started: true + - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified + name: nova-kuttl-api-log + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + service: nova-scheduler + name: nova-kuttl-scheduler-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-scheduler +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-scheduler:current-podified + name: nova-kuttl-scheduler-scheduler + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-metadata + statefulset.kubernetes.io/pod-name: nova-kuttl-metadata-0 + name: nova-kuttl-metadata-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-metadata +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified + name: nova-kuttl-metadata-log + ready: true + started: true + - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified + name: nova-kuttl-metadata-metadata + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + cell: cell0 + service: nova-conductor + statefulset.kubernetes.io/pod-name: nova-kuttl-cell0-conductor-0 + name: nova-kuttl-cell0-conductor-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-cell0-conductor +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified + name: nova-kuttl-cell0-conductor-conductor + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + cell: cell1 + service: nova-conductor + statefulset.kubernetes.io/pod-name: nova-kuttl-cell1-conductor-0 + name: nova-kuttl-cell1-conductor-0 + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-cell1-conductor +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified + name: nova-kuttl-cell1-conductor-conductor + ready: true + started: true +--- +apiVersion: v1 +kind: Service +metadata: + labels: + endpoint: internal + service: nova-api + name: nova-internal + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaAPI + name: nova-kuttl-api +spec: + ports: + - name: nova-internal + selector: + service: nova-api +--- +apiVersion: v1 +kind: Service +metadata: + labels: + endpoint: public + service: nova-api + name: nova-public + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaAPI + name: nova-kuttl-api +spec: + ports: + - name: nova-public + selector: + service: nova-api +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-api.openstack.org/name: nova-kuttl-api + name: nova-kuttl-api-config-data + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaAPI + name: nova-kuttl-api +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-conductor.openstack.org/name: nova-kuttl-cell0-conductor + name: nova-kuttl-cell0-conductor-config-data + namespace: nova-kuttl-default + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell0-conductor +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-conductor.openstack.org/name: nova-kuttl-cell0-conductor + name: nova-kuttl-cell0-conductor-scripts + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell0-conductor +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-conductor.openstack.org/name: nova-kuttl-cell1-conductor + name: nova-kuttl-cell1-conductor-config-data + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell1-conductor +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-conductor.openstack.org/name: nova-kuttl-cell1-conductor + name: nova-kuttl-cell1-conductor-scripts + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell1-conductor +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-metadata.openstack.org/name: nova-kuttl-metadata + name: nova-kuttl-metadata-config-data + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaMetadata + name: nova-kuttl-metadata +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-scheduler.openstack.org/name: nova-kuttl-scheduler + name: nova-kuttl-scheduler-config-data + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaScheduler + name: nova-kuttl-scheduler +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-cell.openstack.org/name: nova-kuttl-cell1 + name: nova-kuttl-cell1-compute-config + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaCell + name: nova-kuttl-cell1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-api + namespace: nova-kuttl-default + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaAPI + name: nova-kuttl-api +spec: + replicas: 1 + selector: + matchLabels: + service: nova-api + serviceName: "" + template: + metadata: + labels: + service: nova-api +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-metadata + namespace: nova-kuttl-default + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaMetadata + name: nova-kuttl-metadata +spec: + replicas: 1 + selector: + matchLabels: + service: nova-metadata + template: + metadata: + labels: + service: nova-metadata +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-scheduler + namespace: nova-kuttl-default + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaScheduler + name: nova-kuttl-scheduler +spec: + replicas: 1 + selector: + matchLabels: + service: nova-scheduler + template: + metadata: + labels: + service: nova-scheduler +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-cell0-conductor + namespace: nova-kuttl-default + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell0-conductor +spec: + replicas: 1 + selector: + matchLabels: + cell: cell0 + service: nova-conductor + template: + metadata: + labels: + cell: cell0 + service: nova-conductor +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-cell1-conductor + namespace: nova-kuttl-default + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell1-conductor +spec: + replicas: 1 + selector: + matchLabels: + cell: cell1 + service: nova-conductor + template: + metadata: + labels: + cell: cell1 + service: nova-conductor +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + labels: + service: nova-conductor + name: nova-kuttl-cell0-db-purge + namespace: nova-kuttl-default +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + completions: 1 + parallelism: 1 + template: + spec: + containers: + - args: + - -c + - /usr/local/bin/kolla_start + command: + - /bin/bash + env: + - name: ARCHIVE_AGE + value: "30" + - name: KOLLA_BOOTSTRAP + value: "true" + - name: KOLLA_CONFIG_STRATEGY + value: COPY_ALWAYS + - name: PURGE_AGE + value: "90" + image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified + name: nova-manage + securityContext: + runAsUser: 42436 + volumeMounts: + - mountPath: /var/lib/openstack/config + name: config-data + - mountPath: /var/lib/openstack/bin + name: scripts + - mountPath: /var/lib/kolla/config_files/config.json + name: config-data + subPath: nova-conductor-dbpurge-config.json + restartPolicy: OnFailure + serviceAccount: nova-nova-kuttl + serviceAccountName: nova-nova-kuttl + volumes: + - name: config-data + secret: + secretName: nova-kuttl-cell0-conductor-config-data + - name: scripts + secret: + secretName: nova-kuttl-cell0-conductor-scripts + schedule: 0 0 * * * + successfulJobsHistoryLimit: 3 + suspend: false +status: {} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + labels: + service: nova-conductor + name: nova-kuttl-cell1-db-purge + namespace: nova-kuttl-default +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + completions: 1 + parallelism: 1 + template: + spec: + containers: + - args: + - -c + - /usr/local/bin/kolla_start + command: + - /bin/bash + env: + - name: ARCHIVE_AGE + value: "30" + - name: KOLLA_BOOTSTRAP + value: "true" + - name: KOLLA_CONFIG_STRATEGY + value: COPY_ALWAYS + - name: PURGE_AGE + value: "90" + image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified + name: nova-manage + securityContext: + runAsUser: 42436 + volumeMounts: + - mountPath: /var/lib/openstack/config + name: config-data + - mountPath: /var/lib/openstack/bin + name: scripts + - mountPath: /var/lib/kolla/config_files/config.json + name: config-data + subPath: nova-conductor-dbpurge-config.json + restartPolicy: OnFailure + serviceAccount: nova-nova-kuttl + serviceAccountName: nova-nova-kuttl + volumes: + - name: config-data + secret: + secretName: nova-kuttl-cell1-conductor-config-data + - name: scripts + secret: + secretName: nova-kuttl-cell1-conductor-scripts + schedule: 0 0 * * * + successfulJobsHistoryLimit: 3 + suspend: false +status: {} diff --git a/test/kuttl/test-suites/default/tests/01-deploy.yaml b/test/kuttl/test-suites/default/tests/01-deploy.yaml new file mode 100644 index 000000000..daae034aa --- /dev/null +++ b/test/kuttl/test-suites/default/tests/01-deploy.yaml @@ -0,0 +1,6 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: Nova +metadata: + name: nova-kuttl +spec: + secret: osp-secret diff --git a/test/kuttl/test-suites/default/tests/02-assert.yaml b/test/kuttl/test-suites/default/tests/02-assert.yaml new file mode 100644 index 000000000..573bd11e4 --- /dev/null +++ b/test/kuttl/test-suites/default/tests/02-assert.yaml @@ -0,0 +1,125 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: Nova +metadata: + finalizers: + - openstack.org/nova + name: nova-kuttl + namespace: nova-kuttl-default +status: + apiServiceReadyCount: 1 + metadataServiceReadyCount: 1 + schedulerServiceReadyCount: 1 + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: Setup complete + reason: Ready + status: "True" + type: KeystoneServiceReady + - message: MariaDBAccount creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady + - message: ' Memcached instance has been provisioned' + reason: Ready + status: "True" + type: MemcachedReady + - message: DB create completed + reason: Ready + status: "True" + type: NovaAPIDBReady + - message: API message bus creation successfully + reason: Ready + status: "True" + type: NovaAPIMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaAPIReady + - message: All DBs created successfully + reason: Ready + status: "True" + type: NovaAllCellDBReady + - message: All NovaCells are ready + reason: Ready + status: "True" + type: NovaAllCellReady + - message: All message busses created successfully + reason: Ready + status: "True" + type: NovaAllCellsMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaMetadataReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaSchedulerReady + - message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - message: Role created + reason: Ready + status: "True" + type: RoleReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-api + name: nova-kuttl-api-0 +status: + containerStatuses: + - name: nova-kuttl-api-api + ready: true + started: true + - name: nova-kuttl-api-log + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-metadata + statefulset.kubernetes.io/pod-name: nova-kuttl-metadata-0 + name: nova-kuttl-metadata-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-metadata +status: + containerStatuses: + - name: nova-kuttl-metadata-log + ready: true + started: true + - name: nova-kuttl-metadata-metadata + ready: true + started: true + +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +namespaced: true +commands: + - command: echo oc get -n $NAMESPACE pod/nova-kuttl-cell1-novncproxy-0 --ignore-not-found + - command: echo oc get -n $NAMESPACE pod/nova-kuttl-cell1-conductor-0--ignore-not-found diff --git a/test/kuttl/test-suites/default/tests/02-delete-cell-nova.yaml b/test/kuttl/test-suites/default/tests/02-delete-cell-nova.yaml new file mode 100644 index 000000000..df6625a64 --- /dev/null +++ b/test/kuttl/test-suites/default/tests/02-delete-cell-nova.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc patch -n nova-kuttl-default nova/nova-kuttl --type='json' -p='[{"op": "remove", "path": "/spec/cellTemplates/cell1"}]' diff --git a/test/kuttl/test-suites/default/tests/03-cleanup-nova.yaml b/test/kuttl/test-suites/default/tests/03-cleanup-nova.yaml new file mode 120000 index 000000000..49b58eee9 --- /dev/null +++ b/test/kuttl/test-suites/default/tests/03-cleanup-nova.yaml @@ -0,0 +1 @@ +00-cleanup-nova.yaml \ No newline at end of file From 2b928418c7173d88d2af49e4d8ce5248cba1da47 Mon Sep 17 00:00:00 2001 From: Kamil Sambor Date: Wed, 25 Sep 2024 14:28:01 +0200 Subject: [PATCH 6/9] Delete additional cell delete job We need to delete only nova finalizer --- .zuul.yaml | 49 -- .../ansible.cfg | 2 - .../inventory.ini | 9 - .../playbooks/deploy-deps.yaml | 18 - .../playbooks/run-kuttl.yaml | 26 - controllers/nova_controller.go | 98 ++- test/functional/base_test.go | 8 - .../common/cleanup-nova.yaml | 7 - .../test-suites/cell-delete-tests/config.yaml | 14 - .../deps/OpenStackControlPlane.yaml | 21 - .../cell-delete-tests/deps/infra.yaml | 32 - .../cell-delete-tests/deps/keystone.yaml | 9 - .../cell-delete-tests/deps/kustomization.yaml | 43 -- .../cell-delete-tests/deps/namespace.yaml | 4 - .../cell-delete-tests/deps/nova.yaml | 7 - .../cell-delete-tests/deps/placement.yaml | 9 - .../tests/00-cleanup-nova.yaml | 1 - .../cell-delete-tests/tests/01-assert.yaml | 623 ------------------ .../cell-delete-tests/tests/01-deploy.yaml | 6 - .../cell-delete-tests/tests/02-assert.yaml | 125 ---- .../tests/02-delete-cell-nova.yaml | 5 - .../tests/03-cleanup-nova.yaml | 1 - 22 files changed, 85 insertions(+), 1032 deletions(-) delete mode 100644 ci/nova-operator-kuttl-cell-delete/ansible.cfg delete mode 100644 ci/nova-operator-kuttl-cell-delete/inventory.ini delete mode 100644 ci/nova-operator-kuttl-cell-delete/playbooks/deploy-deps.yaml delete mode 100644 ci/nova-operator-kuttl-cell-delete/playbooks/run-kuttl.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/common/cleanup-nova.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/config.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/OpenStackControlPlane.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/infra.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/keystone.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/kustomization.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/namespace.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/nova.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/deps/placement.yaml delete mode 120000 test/kuttl/test-suites/cell-delete-tests/tests/00-cleanup-nova.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/tests/01-assert.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/tests/01-deploy.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/tests/02-assert.yaml delete mode 100644 test/kuttl/test-suites/cell-delete-tests/tests/02-delete-cell-nova.yaml delete mode 120000 test/kuttl/test-suites/cell-delete-tests/tests/03-cleanup-nova.yaml diff --git a/.zuul.yaml b/.zuul.yaml index 823c45857..a9e3193f2 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -72,54 +72,6 @@ ip: 172.18.0.5 tenant: ip: 172.19.0.5 -- job: - name: nova-operator-kuttl-cell-delete - parent: nova-operator-base - dependencies: ["openstack-meta-content-provider"] - roles: - - zuul: github.com/openstack-k8s-operators/ci-framework - description: | - This job deploy a basic "Compute Starter Kit" topology - https://www.openstack.org/software/sample-configs#compute-starter-kit - that is the minium set of openstack services required to boot a vm. - pre-run: - - ci/nova-operator-kuttl-cell-delete/playbooks/deploy-deps.yaml - run: - - ci/nova-operator-kuttl-cell-delete/playbooks/run-kuttl.yaml - nodeset: centos-9-medium-crc-extracted-2-39-0-3xl - vars: - collection_namespace_override: "nova-kuttl-cell-delete-tests" - zuul_log_collection: true - extra-vars: - crc_ci_bootstrap_networking: - networks: - default: - range: 192.168.122.0/24 - mtu: 1500 - internal-api: - vlan: 20 - range: 172.17.0.0/24 - storage: - vlan: 21 - range: 172.18.0.0/24 - tenant: - vlan: 22 - range: 172.19.0.0/24 - instances: - controller: - networks: - default: - ip: 192.168.122.11 - crc: - networks: - default: - ip: 192.168.122.10 - internal-api: - ip: 172.17.0.5 - storage: - ip: 172.18.0.5 - tenant: - ip: 172.19.0.5 - job: name: nova-operator-tempest-multinode parent: podified-multinode-edpm-deployment-crc-3comp @@ -259,7 +211,6 @@ - nova-operator-kuttl - nova-operator-tempest-multinode - nova-operator-tempest-multinode-ceph - - nova-operator-kuttl-cell-delete - pragma: implied-branch-matchers: True diff --git a/ci/nova-operator-kuttl-cell-delete/ansible.cfg b/ci/nova-operator-kuttl-cell-delete/ansible.cfg deleted file mode 100644 index 765ca2213..000000000 --- a/ci/nova-operator-kuttl-cell-delete/ansible.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[defaults] -roles_path = /usr/share/ansible/roles:/etc/ansible/roles:../../../ci-framework/roles diff --git a/ci/nova-operator-kuttl-cell-delete/inventory.ini b/ci/nova-operator-kuttl-cell-delete/inventory.ini deleted file mode 100644 index 390656c8f..000000000 --- a/ci/nova-operator-kuttl-cell-delete/inventory.ini +++ /dev/null @@ -1,9 +0,0 @@ -[openshift] -controller ansible_connection=local - -[openshift:vars] -local_nova_operator_basedir= "~/repos/nova-operator" -local_log_dir= "/tmp/nova-operator-kuttl/logs" - -[all:vars] -ansible_user_dir="~/" diff --git a/ci/nova-operator-kuttl-cell-delete/playbooks/deploy-deps.yaml b/ci/nova-operator-kuttl-cell-delete/playbooks/deploy-deps.yaml deleted file mode 100644 index e7ad00d1f..000000000 --- a/ci/nova-operator-kuttl-cell-delete/playbooks/deploy-deps.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- hosts: controller - vars: - nova_operator_basedir: "{{local_nova_operator_basedir | default(ansible_user_dir + '/src/github.com/openstack-k8s-operators/nova-operator')}}" - tasks: - - name: install kuttl test_suite dependencies - environment: - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - PATH: "{{ cifmw_path | default(ansible_env.PATH) }}" - KUTTL_SUITE: "cell-delete-tests" - ansible.builtin.command: - cmd: make kuttl-test-prep - chdir: "{{nova_operator_basedir}}" - register: result - until: result.rc == 0 - retries: 3 - delay: 10 - changed_when: true diff --git a/ci/nova-operator-kuttl-cell-delete/playbooks/run-kuttl.yaml b/ci/nova-operator-kuttl-cell-delete/playbooks/run-kuttl.yaml deleted file mode 100644 index b60878178..000000000 --- a/ci/nova-operator-kuttl-cell-delete/playbooks/run-kuttl.yaml +++ /dev/null @@ -1,26 +0,0 @@ -- hosts: controller - vars: - nova_operator_basedir: "{{local_nova_operator_basedir | default(ansible_user_dir + '/src/github.com/openstack-k8s-operators/nova-operator')}}" - kuttl_log_dir: "{{local_log_dir | default(ansible_user_dir + '/zuul-output/logs/controller')}}" - tasks: - - name: install kuttl test_suite dependencies - environment: - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - PATH: "{{ cifmw_path | default(ansible_env.PATH) }}" - KUTTL_SUITE: "cell-delete-tests" - ansible.builtin.command: - cmd: make kuttl-test-run - chdir: "{{nova_operator_basedir}}" - changed_when: true - register: kuttl_test_run - - name: Create log dir - ansible.builtin.file: - path: "{{kuttl_log_dir}}" - state: directory - mode: "0755" - recurse: yes - - name: save output to file - ansible.builtin.copy: - content: "{{kuttl_test_run.stdout}}" - dest: "{{kuttl_log_dir}}/kuttl-test-run.out" - when: kuttl_test_run.stdout is defined diff --git a/controllers/nova_controller.go b/controllers/nova_controller.go index f58fab6b1..171b4fd7e 100644 --- a/controllers/nova_controller.go +++ b/controllers/nova_controller.go @@ -601,7 +601,6 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul } for _, cr := range novaCellList.Items { - cellDbs := [][]string{} _, ok := instance.Spec.CellTemplates[cr.Spec.CellName] if !ok { err := r.ensureCellDeleted(ctx, h, instance, @@ -610,26 +609,93 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul if err != nil { return ctrl.Result{}, err } - cellDbs = append(cellDbs, []string{novaapi.ServiceName + "-" + cr.Spec.CellName, instance.Spec.CellTemplates[cr.Spec.CellName].CellDatabaseAccount}) + Log.Info("Cell deleted", "cell", cr.Spec.CellName) delete(instance.Status.RegisteredCells, cr.Name) } - // iterate over novaDbs and remove finalizers - for _, novaDb := range cellDbs { - dbName, accountName := novaDb[0], novaDb[1] - db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, h, dbName, accountName, instance.ObjectMeta.Namespace) + + } + + Log.Info("Successfully reconciled") + return ctrl.Result{}, nil +} + +// GetDatabase returns an existing MariaDBDatabase object from the cluster +func GetDatabase(ctx context.Context, + h *helper.Helper, + name string, namespace string, +) (*mariadbv1.MariaDBDatabase, error) { + mariaDBDatabase := &mariadbv1.MariaDBDatabase{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + objectKey := client.ObjectKeyFromObject(mariaDBDatabase) + + err := h.GetClient().Get(ctx, objectKey, mariaDBDatabase) + if err != nil { + return nil, err + } + return mariaDBDatabase, err +} + +func DeleteDatabaseAndAccountFinalizers( + ctx context.Context, + h *helper.Helper, + name string, + accountName string, + namespace string, +) error { + + databaseAccount, err := mariadbv1.GetAccount(ctx, h, accountName, namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } else if err == nil { + if databaseAccount.Spec.Secret != "" { + dbSecret, _, err := secret.GetSecret(ctx, h, databaseAccount.Spec.Secret, namespace) if err != nil && !k8s_errors.IsNotFound(err) { - return ctrl.Result{}, err + return err } - if !k8s_errors.IsNotFound(err) { - if err := db.DeleteFinalizer(ctx, h); err != nil { - return ctrl.Result{}, err + + if err == nil && controllerutil.RemoveFinalizer(dbSecret, h.GetFinalizer()) { + err := h.GetClient().Update(ctx, dbSecret) + if err != nil && !k8s_errors.IsNotFound(err) { + return err } + util.LogForObject(h, fmt.Sprintf("Removed finalizer %s from Secret %s", h.GetFinalizer(), dbSecret.Name), dbSecret) } } + + if controllerutil.RemoveFinalizer(databaseAccount, h.GetFinalizer()) { + err := h.GetClient().Update(ctx, databaseAccount) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + util.LogForObject(h, fmt.Sprintf("Removed finalizer %s from MariaDBAccount %s", h.GetFinalizer(), databaseAccount.Name), databaseAccount) + } } - Log.Info("Successfully reconciled") - return ctrl.Result{}, nil + // also do a delete for "unused" MariaDBAccounts, associated with + // this MariaDBDatabase. + err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers( + ctx, h, name, accountName, namespace, + ) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + + mariaDBDatabase, err := GetDatabase(ctx, h, name, namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } else if err == nil && controllerutil.RemoveFinalizer(mariaDBDatabase, h.GetFinalizer()) { + err := h.GetClient().Update(ctx, mariaDBDatabase) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + util.LogForObject(h, fmt.Sprintf("Removed finalizer %s from MariaDBDatabase %s", h.GetFinalizer(), mariaDBDatabase.Spec.Name), mariaDBDatabase) + } + + return nil } func (r *NovaReconciler) ensureCellDeleted( @@ -665,11 +731,14 @@ func (r *NovaReconciler) ensureCellDeleted( "cell", cell) return nil } + err = r.Client.Delete(ctx, cell) if err != nil && k8s_errors.IsNotFound(err) { return err } + dbName, accountName := novaapi.ServiceName+"-"+cell.Spec.CellName, cell.Spec.CellDatabaseAccount + configHash, scriptName, configName, err := r.ensureNovaManageJobSecret(ctx, h, instance, cell0, topLevelSecret, APIDatabaseHostname, apiTransportURL, apiDB) if err != nil { @@ -687,7 +756,7 @@ func (r *NovaReconciler) ensureCellDeleted( job := job.NewJob( jobDef, cell.Name+"-cell-delete", instance.Spec.PreserveJobs, r.RequeueTimeout, - instance.Status.RegisteredCells[cell.Name]) + inputHash) _, err = job.DoJob(ctx, h) if err != nil { @@ -714,6 +783,9 @@ func (r *NovaReconciler) ensureCellDeleted( }, } err = r.Client.Delete(ctx, transportURL) + + err = DeleteDatabaseAndAccountFinalizers(ctx, h, dbName, accountName, instance.ObjectMeta.Namespace) + if err != nil { return err } diff --git a/test/functional/base_test.go b/test/functional/base_test.go index b39a87bd9..34624170c 100644 --- a/test/functional/base_test.go +++ b/test/functional/base_test.go @@ -214,14 +214,6 @@ func NovaConductorConditionGetter(name types.NamespacedName) condition.Condition return instance.Status.Conditions } -func AssertConductorDoesNotExist(name types.NamespacedName) { - instance := &novav1.NovaConductor{} - Eventually(func(g Gomega) { - err := k8sClient.Get(ctx, name, instance) - g.Expect(k8s_errors.IsNotFound(err)).To(BeTrue()) - }, timeout, interval).Should(Succeed()) -} - func CreateNovaMessageBusSecret(cell CellNames) *corev1.Secret { s := th.CreateSecret( types.NamespacedName{Namespace: cell.CellCRName.Namespace, Name: fmt.Sprintf("%s-secret", cell.TransportURLName.Name)}, diff --git a/test/kuttl/test-suites/cell-delete-tests/common/cleanup-nova.yaml b/test/kuttl/test-suites/cell-delete-tests/common/cleanup-nova.yaml deleted file mode 100644 index 55caf71d6..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/common/cleanup-nova.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -delete: -- apiVersion: nova.openstack.org/v1beta1 - kind: Nova - name: nova-kuttl-cell - namespace: nova-kuttl-cell-delete-tests diff --git a/test/kuttl/test-suites/cell-delete-tests/config.yaml b/test/kuttl/test-suites/cell-delete-tests/config.yaml deleted file mode 100644 index 52464bafd..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestSuite -reportFormat: JSON -reportName: kuttl-default-results -namespace: nova-kuttl-cell-delete-tests -# we could set this lower, but the initial image pull can take a while -timeout: 300 -parallel: 1 -skipDelete: true -testDirs: - - test/kuttl/test-suites/cell-delete-tests/ -suppress: - - events -artifactsDir: test/kuttl/test-suites/cell-delete-tests/output diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/OpenStackControlPlane.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/OpenStackControlPlane.yaml deleted file mode 100644 index f76d003d9..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/deps/OpenStackControlPlane.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: core.openstack.org/v1beta1 -kind: OpenStackControlPlane -metadata: - name: openstack -spec: - storageClass: "crc-csi-hostpath-provisioner" - tls: - ingress: - enabled: true - podLevel: - enabled: false - ironic: - enabled: false - template: - ironicConductors: [] - manila: - enabled: false - template: - manilaShares: {} - horizon: - enabled: false diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/infra.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/infra.yaml deleted file mode 100644 index 1dd64e878..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/deps/infra.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: core.openstack.org/v1beta1 -kind: OpenStackControlPlane -metadata: - name: openstack -spec: - mariadb: - enabled: false - galera: - enabled: true - templates: - openstack: - storageRequest: 500M - openstack-cell1: - storageRequest: 500M - rabbitmq: - templates: - rabbitmq: - replicas: 1 - rabbitmq-cell1: - replicas: 1 - memcached: - templates: - memcached: - replicas: 1 - ovn: - enabled: false - template: - ovnController: - external-ids: - ovn-encap-type: geneve - ovs: - enabled: false diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/keystone.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/keystone.yaml deleted file mode 100644 index 97f568e7b..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/deps/keystone.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: core.openstack.org/v1beta1 -kind: OpenStackControlPlane -metadata: - name: openstack -spec: - keystone: - template: - databaseInstance: openstack - secret: osp-secret diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/kustomization.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/kustomization.yaml deleted file mode 100644 index 0a0ec3454..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/deps/kustomization.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: nova-kuttl-cell-delete-tests - -secretGenerator: -- literals: - - AdminPassword=password - - DbRootPassword=password - - DatabasePassword=password - - KeystoneDatabasePassword=password - - PlacementPassword=password - - PlacementDatabasePassword=password - - GlancePassword=password - - GlanceDatabasePassword=password - - NeutronPassword=password - - NeutronDatabasePassword=password - - NovaPassword=password - - NovaAPIDatabasePassword=password - - NovaCell0DatabasePassword=password - - NovaCell1DatabasePassword=password - - MetadataSecret=42 - name: osp-secret -generatorOptions: - disableNameSuffixHash: true - labels: - type: osp-secret - -resources: -- namespace.yaml -- OpenStackControlPlane.yaml - -patches: -- patch: |- - apiVersion: core.openstack.org/v1beta1 - kind: OpenStackControlPlane - metadata: - name: openstack - spec: - secret: osp-secret -- path: infra.yaml -- path: keystone.yaml -- path: placement.yaml -- path: nova.yaml diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/namespace.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/namespace.yaml deleted file mode 100644 index 68e0205cd..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/deps/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: nova-kuttl-cell-delete-tests diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/nova.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/nova.yaml deleted file mode 100644 index 289dfc1b2..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/deps/nova.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: core.openstack.org/v1beta1 -kind: OpenStackControlPlane -metadata: - name: openstack -spec: - nova: - enabled: false diff --git a/test/kuttl/test-suites/cell-delete-tests/deps/placement.yaml b/test/kuttl/test-suites/cell-delete-tests/deps/placement.yaml deleted file mode 100644 index ed54e8c42..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/deps/placement.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: core.openstack.org/v1beta1 -kind: OpenStackControlPlane -metadata: - name: openstack -spec: - placement: - template: - databaseInstance: openstack - secret: osp-secret diff --git a/test/kuttl/test-suites/cell-delete-tests/tests/00-cleanup-nova.yaml b/test/kuttl/test-suites/cell-delete-tests/tests/00-cleanup-nova.yaml deleted file mode 120000 index 23d51de43..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/tests/00-cleanup-nova.yaml +++ /dev/null @@ -1 +0,0 @@ -../common/cleanup-nova.yaml \ No newline at end of file diff --git a/test/kuttl/test-suites/cell-delete-tests/tests/01-assert.yaml b/test/kuttl/test-suites/cell-delete-tests/tests/01-assert.yaml deleted file mode 100644 index 54312a821..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/tests/01-assert.yaml +++ /dev/null @@ -1,623 +0,0 @@ -apiVersion: nova.openstack.org/v1beta1 -kind: Nova -metadata: - finalizers: - - openstack.org/nova - name: nova-kuttl-cell - namespace: nova-kuttl-cell-delete-tests -spec: - apiDatabaseInstance: openstack - apiDatabaseAccount: nova-api - apiMessageBusInstance: rabbitmq - apiServiceTemplate: - customServiceConfig: "" - replicas: 1 - resources: {} - cellTemplates: - cell0: - cellDatabaseInstance: openstack - cellDatabaseAccount: nova-cell0 - cellMessageBusInstance: rabbitmq - conductorServiceTemplate: - customServiceConfig: "" - replicas: 1 - resources: {} - hasAPIAccess: true - metadataServiceTemplate: - enabled: false - noVNCProxyServiceTemplate: - enabled: false - cell1: - cellDatabaseInstance: openstack-cell1 - cellDatabaseAccount: nova-cell1 - cellMessageBusInstance: rabbitmq-cell1 - conductorServiceTemplate: - customServiceConfig: "" - replicas: 1 - resources: {} - hasAPIAccess: true - metadataServiceTemplate: - enabled: false - noVNCProxyServiceTemplate: - enabled: true - customServiceConfig: "" - replicas: 1 - resources: {} - keystoneInstance: keystone - metadataServiceTemplate: - enabled: true - customServiceConfig: "" - replicas: 1 - resources: {} - passwordSelectors: - metadataSecret: MetadataSecret - service: NovaPassword - schedulerServiceTemplate: - customServiceConfig: "" - replicas: 1 - resources: {} - secret: osp-secret - serviceUser: nova -status: - apiServiceReadyCount: 1 - metadataServiceReadyCount: 1 - schedulerServiceReadyCount: 1 - conditions: - - message: Setup complete - reason: Ready - status: "True" - type: Ready - - message: Input data complete - reason: Ready - status: "True" - type: InputReady - - message: Setup complete - reason: Ready - status: "True" - type: KeystoneServiceReady - - message: MariaDBAccount creation complete - reason: Ready - status: "True" - type: MariaDBAccountReady - - message: ' Memcached instance has been provisioned' - reason: Ready - status: "True" - type: MemcachedReady - - message: DB create completed - reason: Ready - status: "True" - type: NovaAPIDBReady - - message: API message bus creation successfully - reason: Ready - status: "True" - type: NovaAPIMQReady - - message: Setup complete - reason: Ready - status: "True" - type: NovaAPIReady - - message: All DBs created successfully - reason: Ready - status: "True" - type: NovaAllCellDBReady - - message: All NovaCells are ready - reason: Ready - status: "True" - type: NovaAllCellReady - - message: All message busses created successfully - reason: Ready - status: "True" - type: NovaAllCellsMQReady - - message: Setup complete - reason: Ready - status: "True" - type: NovaMetadataReady - - message: Setup complete - reason: Ready - status: "True" - type: NovaSchedulerReady - - message: RoleBinding created - reason: Ready - status: "True" - type: RoleBindingReady - - message: Role created - reason: Ready - status: "True" - type: RoleReady - - message: ServiceAccount created - reason: Ready - status: "True" - type: ServiceAccountReady ---- -apiVersion: v1 -kind: Pod -metadata: - annotations: - openshift.io/scc: anyuid - labels: - service: nova-api - name: nova-kuttl-cell-api-0 -status: - containerStatuses: - - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified - name: nova-kuttl-cell-api-api - ready: true - started: true - - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified - name: nova-kuttl-cell-api-log - ready: true - started: true ---- -apiVersion: v1 -kind: Pod -metadata: - labels: - service: nova-scheduler - name: nova-kuttl-cell-scheduler-0 - ownerReferences: - - apiVersion: apps/v1 - blockOwnerDeletion: true - controller: true - kind: StatefulSet - name: nova-kuttl-cell-scheduler -status: - containerStatuses: - - image: quay.io/podified-antelope-centos9/openstack-nova-scheduler:current-podified - name: nova-kuttl-cell-scheduler-scheduler - ready: true - started: true ---- -apiVersion: v1 -kind: Pod -metadata: - annotations: - openshift.io/scc: anyuid - labels: - service: nova-metadata - statefulset.kubernetes.io/pod-name: nova-kuttl-cell-metadata-0 - name: nova-kuttl-cell-metadata-0 - ownerReferences: - - apiVersion: apps/v1 - blockOwnerDeletion: true - controller: true - kind: StatefulSet - name: nova-kuttl-cell-metadata -status: - containerStatuses: - - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified - name: nova-kuttl-cell-metadata-log - ready: true - started: true - - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified - name: nova-kuttl-cell-metadata-metadata - ready: true - started: true ---- -apiVersion: v1 -kind: Pod -metadata: - annotations: - openshift.io/scc: anyuid - labels: - cell: cell0 - service: nova-conductor - statefulset.kubernetes.io/pod-name: nova-kuttl-cell-cell0-conductor-0 - name: nova-kuttl-cell-cell0-conductor-0 - ownerReferences: - - apiVersion: apps/v1 - blockOwnerDeletion: true - controller: true - kind: StatefulSet - name: nova-kuttl-cell-cell0-conductor -status: - containerStatuses: - - image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified - name: nova-kuttl-cell-cell0-conductor-conductor - ready: true - started: true ---- -apiVersion: v1 -kind: Pod -metadata: - annotations: - openshift.io/scc: anyuid - labels: - cell: cell1 - service: nova-conductor - statefulset.kubernetes.io/pod-name: nova-kuttl-cell-cell1-conductor-0 - name: nova-kuttl-cell-cell1-conductor-0 - ownerReferences: - - blockOwnerDeletion: true - controller: true - kind: StatefulSet - name: nova-kuttl-cell-cell1-conductor -status: - containerStatuses: - - image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified - name: nova-kuttl-cell-cell1-conductor-conductor - ready: true - started: true ---- -apiVersion: v1 -kind: Service -metadata: - labels: - endpoint: internal - service: nova-api - name: nova-internal - ownerReferences: - - blockOwnerDeletion: true - controller: true - kind: NovaAPI - name: nova-kuttl-cell-api -spec: - ports: - - name: nova-internal - selector: - service: nova-api ---- -apiVersion: v1 -kind: Service -metadata: - labels: - endpoint: public - service: nova-api - name: nova-public - ownerReferences: - - blockOwnerDeletion: true - controller: true - kind: NovaAPI - name: nova-kuttl-cell-api -spec: - ports: - - name: nova-public - selector: - service: nova-api ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - nova-api.openstack.org/name: nova-kuttl-cell-api - name: nova-kuttl-cell-api-config-data - ownerReferences: - - blockOwnerDeletion: true - controller: true - kind: NovaAPI - name: nova-kuttl-cell-api ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - nova-conductor.openstack.org/name: nova-kuttl-cell-cell0-conductor - name: nova-kuttl-cell-cell0-conductor-config-data - namespace: nova-kuttl-cell-delete-tests - ownerReferences: - - blockOwnerDeletion: true - controller: true - kind: NovaConductor - name: nova-kuttl-cell-cell0-conductor ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - nova-conductor.openstack.org/name: nova-kuttl-cell-cell0-conductor - name: nova-kuttl-cell-cell0-conductor-scripts - ownerReferences: - - blockOwnerDeletion: true - controller: true - kind: NovaConductor - name: nova-kuttl-cell-cell0-conductor ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - nova-conductor.openstack.org/name: nova-kuttl-cell-cell1-conductor - name: nova-kuttl-cell-cell1-conductor-config-data - ownerReferences: - - blockOwnerDeletion: true - controller: true - kind: NovaConductor - name: nova-kuttl-cell-cell1-conductor ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - nova-conductor.openstack.org/name: nova-kuttl-cell-cell1-conductor - name: nova-kuttl-cell-cell1-conductor-scripts - ownerReferences: - - blockOwnerDeletion: true - controller: true - kind: NovaConductor - name: nova-kuttl-cell-cell1-conductor ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - nova-metadata.openstack.org/name: nova-kuttl-cell-metadata - name: nova-kuttl-cell-metadata-config-data - ownerReferences: - - blockOwnerDeletion: true - controller: true - kind: NovaMetadata - name: nova-kuttl-cell-metadata ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - nova-scheduler.openstack.org/name: nova-kuttl-cell-scheduler - name: nova-kuttl-cell-scheduler-config-data - ownerReferences: - - blockOwnerDeletion: true - controller: true - kind: NovaScheduler - name: nova-kuttl-cell-scheduler ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - nova-cell.openstack.org/name: nova-kuttl-cell-cell1 - name: nova-kuttl-cell-cell1-compute-config - ownerReferences: - - apiVersion: nova.openstack.org/v1beta1 - blockOwnerDeletion: true - controller: true - kind: NovaCell - name: nova-kuttl-cell-cell1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: nova-kuttl-cell-api - namespace: nova-kuttl-cell-delete-tests - ownerReferences: - - apiVersion: nova.openstack.org/v1beta1 - blockOwnerDeletion: true - controller: true - kind: NovaAPI - name: nova-kuttl-cell-api -spec: - replicas: 1 - selector: - matchLabels: - service: nova-api - serviceName: "" - template: - metadata: - labels: - service: nova-api -status: - availableReplicas: 1 - currentReplicas: 1 - readyReplicas: 1 - replicas: 1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: nova-kuttl-cell-metadata - namespace: nova-kuttl-cell-delete-tests - ownerReferences: - - apiVersion: nova.openstack.org/v1beta1 - blockOwnerDeletion: true - controller: true - kind: NovaMetadata - name: nova-kuttl-cell-metadata -spec: - replicas: 1 - selector: - matchLabels: - service: nova-metadata - template: - metadata: - labels: - service: nova-metadata -status: - availableReplicas: 1 - currentReplicas: 1 - readyReplicas: 1 - replicas: 1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: nova-kuttl-cell-scheduler - namespace: nova-kuttl-cell-delete-tests - ownerReferences: - - apiVersion: nova.openstack.org/v1beta1 - blockOwnerDeletion: true - controller: true - kind: NovaScheduler - name: nova-kuttl-cell-scheduler -spec: - replicas: 1 - selector: - matchLabels: - service: nova-scheduler - template: - metadata: - labels: - service: nova-scheduler -status: - availableReplicas: 1 - currentReplicas: 1 - readyReplicas: 1 - replicas: 1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: nova-kuttl-cell-cell0-conductor - namespace: nova-kuttl-cell-delete-tests - ownerReferences: - - apiVersion: nova.openstack.org/v1beta1 - blockOwnerDeletion: true - controller: true - kind: NovaConductor - name: nova-kuttl-cell-cell0-conductor -spec: - replicas: 1 - selector: - matchLabels: - cell: cell0 - service: nova-conductor - template: - metadata: - labels: - cell: cell0 - service: nova-conductor -status: - availableReplicas: 1 - currentReplicas: 1 - readyReplicas: 1 - replicas: 1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: nova-kuttl-cell-cell1-conductor - namespace: nova-kuttl-cell-delete-tests - ownerReferences: - - apiVersion: nova.openstack.org/v1beta1 - blockOwnerDeletion: true - controller: true - kind: NovaConductor - name: nova-kuttl-cell-cell1-conductor -spec: - replicas: 1 - selector: - matchLabels: - cell: cell1 - service: nova-conductor - template: - metadata: - labels: - cell: cell1 - service: nova-conductor -status: - availableReplicas: 1 - currentReplicas: 1 - readyReplicas: 1 - replicas: 1 ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - labels: - service: nova-conductor - name: nova-kuttl-cell-cell0-db-purge - namespace: nova-kuttl-cell-delete-tests -spec: - concurrencyPolicy: Forbid - failedJobsHistoryLimit: 1 - jobTemplate: - spec: - completions: 1 - parallelism: 1 - template: - spec: - containers: - - args: - - -c - - /usr/local/bin/kolla_start - command: - - /bin/bash - env: - - name: ARCHIVE_AGE - value: "30" - - name: KOLLA_BOOTSTRAP - value: "true" - - name: KOLLA_CONFIG_STRATEGY - value: COPY_ALWAYS - - name: PURGE_AGE - value: "90" - image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified - name: nova-manage - securityContext: - runAsUser: 42436 - volumeMounts: - - mountPath: /var/lib/openstack/config - name: config-data - - mountPath: /var/lib/openstack/bin - name: scripts - - mountPath: /var/lib/kolla/config_files/config.json - name: config-data - subPath: nova-conductor-dbpurge-config.json - restartPolicy: OnFailure - serviceAccount: nova-nova-kuttl-cell - serviceAccountName: nova-nova-kuttl-cell - volumes: - - name: config-data - secret: - secretName: nova-kuttl-cell-cell0-conductor-config-data - - name: scripts - secret: - secretName: nova-kuttl-cell-cell0-conductor-scripts - schedule: 0 0 * * * - successfulJobsHistoryLimit: 3 - suspend: false -status: {} ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - labels: - service: nova-conductor - name: nova-kuttl-cell-cell1-db-purge - namespace: nova-kuttl-cell-delete-tests -spec: - concurrencyPolicy: Forbid - failedJobsHistoryLimit: 1 - jobTemplate: - spec: - completions: 1 - parallelism: 1 - template: - spec: - containers: - - args: - - -c - - /usr/local/bin/kolla_start - command: - - /bin/bash - env: - - name: ARCHIVE_AGE - value: "30" - - name: KOLLA_BOOTSTRAP - value: "true" - - name: KOLLA_CONFIG_STRATEGY - value: COPY_ALWAYS - - name: PURGE_AGE - value: "90" - image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified - name: nova-manage - securityContext: - runAsUser: 42436 - volumeMounts: - - mountPath: /var/lib/openstack/config - name: config-data - - mountPath: /var/lib/openstack/bin - name: scripts - - mountPath: /var/lib/kolla/config_files/config.json - name: config-data - subPath: nova-conductor-dbpurge-config.json - restartPolicy: OnFailure - serviceAccount: nova-nova-kuttl-cell - serviceAccountName: nova-nova-kuttl-cell - volumes: - - name: config-data - secret: - secretName: nova-kuttl-cell-cell1-conductor-config-data - - name: scripts - secret: - secretName: nova-kuttl-cell-cell1-conductor-scripts - schedule: 0 0 * * * - successfulJobsHistoryLimit: 3 - suspend: false -status: {} diff --git a/test/kuttl/test-suites/cell-delete-tests/tests/01-deploy.yaml b/test/kuttl/test-suites/cell-delete-tests/tests/01-deploy.yaml deleted file mode 100644 index 8c1addcd5..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/tests/01-deploy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: nova.openstack.org/v1beta1 -kind: Nova -metadata: - name: nova-kuttl-cell -spec: - secret: osp-secret diff --git a/test/kuttl/test-suites/cell-delete-tests/tests/02-assert.yaml b/test/kuttl/test-suites/cell-delete-tests/tests/02-assert.yaml deleted file mode 100644 index a0151166e..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/tests/02-assert.yaml +++ /dev/null @@ -1,125 +0,0 @@ -apiVersion: nova.openstack.org/v1beta1 -kind: Nova -metadata: - finalizers: - - openstack.org/nova - name: nova-kuttl-cell - namespace: nova-kuttl-cell-delete-tests -status: - apiServiceReadyCount: 1 - metadataServiceReadyCount: 1 - schedulerServiceReadyCount: 1 - conditions: - - message: Setup complete - reason: Ready - status: "True" - type: Ready - - message: Input data complete - reason: Ready - status: "True" - type: InputReady - - message: Setup complete - reason: Ready - status: "True" - type: KeystoneServiceReady - - message: MariaDBAccount creation complete - reason: Ready - status: "True" - type: MariaDBAccountReady - - message: ' Memcached instance has been provisioned' - reason: Ready - status: "True" - type: MemcachedReady - - message: DB create completed - reason: Ready - status: "True" - type: NovaAPIDBReady - - message: API message bus creation successfully - reason: Ready - status: "True" - type: NovaAPIMQReady - - message: Setup complete - reason: Ready - status: "True" - type: NovaAPIReady - - message: All DBs created successfully - reason: Ready - status: "True" - type: NovaAllCellDBReady - - message: All NovaCells are ready - reason: Ready - status: "True" - type: NovaAllCellReady - - message: All message busses created successfully - reason: Ready - status: "True" - type: NovaAllCellsMQReady - - message: Setup complete - reason: Ready - status: "True" - type: NovaMetadataReady - - message: Setup complete - reason: Ready - status: "True" - type: NovaSchedulerReady - - message: RoleBinding created - reason: Ready - status: "True" - type: RoleBindingReady - - message: Role created - reason: Ready - status: "True" - type: RoleReady - - message: ServiceAccount created - reason: Ready - status: "True" - type: ServiceAccountReady ---- -apiVersion: v1 -kind: Pod -metadata: - annotations: - openshift.io/scc: anyuid - labels: - service: nova-api - name: nova-kuttl-cell-api-0 -status: - containerStatuses: - - name: nova-kuttl-cell-api-api - ready: true - started: true - - name: nova-kuttl-cell-api-log - ready: true - started: true ---- -apiVersion: v1 -kind: Pod -metadata: - annotations: - openshift.io/scc: anyuid - labels: - service: nova-metadata - statefulset.kubernetes.io/pod-name: nova-kuttl-cell-metadata-0 - name: nova-kuttl-cell-metadata-0 - ownerReferences: - - apiVersion: apps/v1 - blockOwnerDeletion: true - controller: true - kind: StatefulSet - name: nova-kuttl-cell-metadata -status: - containerStatuses: - - name: nova-kuttl-cell-metadata-log - ready: true - started: true - - name: nova-kuttl-cell-metadata-metadata - ready: true - started: true - ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -namespaced: true -commands: - - command: echo oc get -n $NAMESPACE pod/nova-cell1-novncproxy-0 --ignore-not-found - - command: echo oc get -n $NAMESPACE pod/nova-cell1-conductor-0--ignore-not-found diff --git a/test/kuttl/test-suites/cell-delete-tests/tests/02-delete-cell-nova.yaml b/test/kuttl/test-suites/cell-delete-tests/tests/02-delete-cell-nova.yaml deleted file mode 100644 index f1b5d904e..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/tests/02-delete-cell-nova.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: | - oc patch -n nova-kuttl-cell-delete-tests nova/nova-kuttl-cell --type='json' -p='[{"op": "remove", "path": "/spec/cellTemplates/cell1"}]' diff --git a/test/kuttl/test-suites/cell-delete-tests/tests/03-cleanup-nova.yaml b/test/kuttl/test-suites/cell-delete-tests/tests/03-cleanup-nova.yaml deleted file mode 120000 index 49b58eee9..000000000 --- a/test/kuttl/test-suites/cell-delete-tests/tests/03-cleanup-nova.yaml +++ /dev/null @@ -1 +0,0 @@ -00-cleanup-nova.yaml \ No newline at end of file From 83c08b0c8b79c874018b8d99866edb1aaf0b73da Mon Sep 17 00:00:00 2001 From: Kamil Sambor Date: Thu, 26 Sep 2024 19:45:03 +0200 Subject: [PATCH 7/9] Reorganize kuttl tests for nova-cell-deletion --- controllers/nova_controller.go | 10 ++++------ .../default/{tests => cell-tests}/00-cleanup-nova.yaml | 0 .../default/{tests => cell-tests}/01-assert.yaml | 0 .../default/{tests => cell-tests}/01-deploy.yaml | 0 .../default/{tests => cell-tests}/02-assert.yaml | 9 ++++++--- .../{tests => cell-tests}/02-delete-cell-nova.yaml | 0 .../default/{tests => cell-tests}/03-cleanup-nova.yaml | 0 7 files changed, 10 insertions(+), 9 deletions(-) rename test/kuttl/test-suites/default/{tests => cell-tests}/00-cleanup-nova.yaml (100%) rename test/kuttl/test-suites/default/{tests => cell-tests}/01-assert.yaml (100%) rename test/kuttl/test-suites/default/{tests => cell-tests}/01-deploy.yaml (100%) rename test/kuttl/test-suites/default/{tests => cell-tests}/02-assert.yaml (92%) rename test/kuttl/test-suites/default/{tests => cell-tests}/02-delete-cell-nova.yaml (100%) rename test/kuttl/test-suites/default/{tests => cell-tests}/03-cleanup-nova.yaml (100%) diff --git a/controllers/nova_controller.go b/controllers/nova_controller.go index 171b4fd7e..bebab1d8d 100644 --- a/controllers/nova_controller.go +++ b/controllers/nova_controller.go @@ -763,12 +763,6 @@ func (r *NovaReconciler) ensureCellDeleted( return err } - // Delete secrets - dbSecret := fmt.Sprintf("%s-db-secret", cell.Name) - err = secret.DeleteSecretsWithName(ctx, h, dbSecret, instance.Namespace) - if err != nil { - return err - } secretName := getNovaCellCRName(instance.Name, cellName) err = secret.DeleteSecretsWithName(ctx, h, secretName, instance.Namespace) if err != nil { @@ -784,6 +778,10 @@ func (r *NovaReconciler) ensureCellDeleted( } err = r.Client.Delete(ctx, transportURL) + if err != nil { + return err + } + err = DeleteDatabaseAndAccountFinalizers(ctx, h, dbName, accountName, instance.ObjectMeta.Namespace) if err != nil { diff --git a/test/kuttl/test-suites/default/tests/00-cleanup-nova.yaml b/test/kuttl/test-suites/default/cell-tests/00-cleanup-nova.yaml similarity index 100% rename from test/kuttl/test-suites/default/tests/00-cleanup-nova.yaml rename to test/kuttl/test-suites/default/cell-tests/00-cleanup-nova.yaml diff --git a/test/kuttl/test-suites/default/tests/01-assert.yaml b/test/kuttl/test-suites/default/cell-tests/01-assert.yaml similarity index 100% rename from test/kuttl/test-suites/default/tests/01-assert.yaml rename to test/kuttl/test-suites/default/cell-tests/01-assert.yaml diff --git a/test/kuttl/test-suites/default/tests/01-deploy.yaml b/test/kuttl/test-suites/default/cell-tests/01-deploy.yaml similarity index 100% rename from test/kuttl/test-suites/default/tests/01-deploy.yaml rename to test/kuttl/test-suites/default/cell-tests/01-deploy.yaml diff --git a/test/kuttl/test-suites/default/tests/02-assert.yaml b/test/kuttl/test-suites/default/cell-tests/02-assert.yaml similarity index 92% rename from test/kuttl/test-suites/default/tests/02-assert.yaml rename to test/kuttl/test-suites/default/cell-tests/02-assert.yaml index 573bd11e4..807eefe98 100644 --- a/test/kuttl/test-suites/default/tests/02-assert.yaml +++ b/test/kuttl/test-suites/default/cell-tests/02-assert.yaml @@ -119,7 +119,10 @@ status: --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert -namespaced: true commands: - - command: echo oc get -n $NAMESPACE pod/nova-kuttl-cell1-novncproxy-0 --ignore-not-found - - command: echo oc get -n $NAMESPACE pod/nova-kuttl-cell1-conductor-0--ignore-not-found + - script: | + oc get -n nova-kuttl-default pod/nova-kuttl-cell1-novncproxy-0 || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default pod/nova-kuttl-cell1-conductor-0 || exit 0 + exit 1 diff --git a/test/kuttl/test-suites/default/tests/02-delete-cell-nova.yaml b/test/kuttl/test-suites/default/cell-tests/02-delete-cell-nova.yaml similarity index 100% rename from test/kuttl/test-suites/default/tests/02-delete-cell-nova.yaml rename to test/kuttl/test-suites/default/cell-tests/02-delete-cell-nova.yaml diff --git a/test/kuttl/test-suites/default/tests/03-cleanup-nova.yaml b/test/kuttl/test-suites/default/cell-tests/03-cleanup-nova.yaml similarity index 100% rename from test/kuttl/test-suites/default/tests/03-cleanup-nova.yaml rename to test/kuttl/test-suites/default/cell-tests/03-cleanup-nova.yaml From 290ffa2b2c578ce88f02e2a5391920fb5e00a30a Mon Sep 17 00:00:00 2001 From: Balazs Gibizer Date: Fri, 27 Sep 2024 14:28:09 +0200 Subject: [PATCH 8/9] Delete MariaDBAccount and Database Both CR is created by the nova controller for the given cell, so when that cell is deleted the CRs also needs to be deleted. We also need to delete the secret connected to the MariaDBAccount as mariadb-operator does not delete it today. It also added graceful handling of NotFound during resource deletion and refactored the cleanup logic to avoid bugs if the delete needs to be retried. This now depends on the https://github.com/openstack-k8s-operators/mariadb-operator/pull/268 instead of copying that change. --- controllers/nova_controller.go | 143 +++++++++---------- go.mod | 2 +- go.sum | 4 +- test/functional/nova_reconfiguration_test.go | 6 +- 4 files changed, 78 insertions(+), 77 deletions(-) diff --git a/controllers/nova_controller.go b/controllers/nova_controller.go index bebab1d8d..096eb25af 100644 --- a/controllers/nova_controller.go +++ b/controllers/nova_controller.go @@ -619,82 +619,43 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul return ctrl.Result{}, nil } -// GetDatabase returns an existing MariaDBDatabase object from the cluster -func GetDatabase(ctx context.Context, - h *helper.Helper, - name string, namespace string, -) (*mariadbv1.MariaDBDatabase, error) { - mariaDBDatabase := &mariadbv1.MariaDBDatabase{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - } - objectKey := client.ObjectKeyFromObject(mariaDBDatabase) - - err := h.GetClient().Get(ctx, objectKey, mariaDBDatabase) - if err != nil { - return nil, err - } - return mariaDBDatabase, err -} - -func DeleteDatabaseAndAccountFinalizers( +func (r *NovaReconciler) ensureAccountDeletedIfOwned( ctx context.Context, h *helper.Helper, - name string, + instance *novav1.Nova, accountName string, - namespace string, ) error { + Log := r.GetLogger(ctx) - databaseAccount, err := mariadbv1.GetAccount(ctx, h, accountName, namespace) + account, err := mariadbv1.GetAccount(ctx, h, accountName, instance.Namespace) if err != nil && !k8s_errors.IsNotFound(err) { return err - } else if err == nil { - if databaseAccount.Spec.Secret != "" { - dbSecret, _, err := secret.GetSecret(ctx, h, databaseAccount.Spec.Secret, namespace) - if err != nil && !k8s_errors.IsNotFound(err) { - return err - } - - if err == nil && controllerutil.RemoveFinalizer(dbSecret, h.GetFinalizer()) { - err := h.GetClient().Update(ctx, dbSecret) - if err != nil && !k8s_errors.IsNotFound(err) { - return err - } - util.LogForObject(h, fmt.Sprintf("Removed finalizer %s from Secret %s", h.GetFinalizer(), dbSecret.Name), dbSecret) - } - } + } + if k8s_errors.IsNotFound(err) { + // Nothing to delete + return nil + } - if controllerutil.RemoveFinalizer(databaseAccount, h.GetFinalizer()) { - err := h.GetClient().Update(ctx, databaseAccount) - if err != nil && !k8s_errors.IsNotFound(err) { - return err - } - util.LogForObject(h, fmt.Sprintf("Removed finalizer %s from MariaDBAccount %s", h.GetFinalizer(), databaseAccount.Name), databaseAccount) - } + // If it is not created by us, we don't clean it up + if !OwnedBy(account, instance) { + Log.Info("MariaDBAccount in not owned by Nova, not deleting", "account", account) + return nil } - // also do a delete for "unused" MariaDBAccounts, associated with - // this MariaDBDatabase. - err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers( - ctx, h, name, accountName, namespace, - ) + // NOTE(gibi): We need to delete the Secret first and then the Account + // otherwise we cannot retry the Secret deletion when the Account is + // gone as we will not know the name of the Secret. This logic should + // be moved to the mariadb-operator. + err = secret.DeleteSecretsWithName(ctx, h, account.Spec.Secret, instance.Namespace) if err != nil && !k8s_errors.IsNotFound(err) { return err } - - mariaDBDatabase, err := GetDatabase(ctx, h, name, namespace) + err = r.Client.Delete(ctx, account) if err != nil && !k8s_errors.IsNotFound(err) { return err - } else if err == nil && controllerutil.RemoveFinalizer(mariaDBDatabase, h.GetFinalizer()) { - err := h.GetClient().Update(ctx, mariaDBDatabase) - if err != nil && !k8s_errors.IsNotFound(err) { - return err - } - util.LogForObject(h, fmt.Sprintf("Removed finalizer %s from MariaDBDatabase %s", h.GetFinalizer(), mariaDBDatabase.Spec.Name), mariaDBDatabase) } + Log.Info("Deleted MariaDBAccount", "account", account) return nil } @@ -718,7 +679,9 @@ func (r *NovaReconciler) ensureCellDeleted( err := r.Client.Get(ctx, fullCellName, cell) if k8s_errors.IsNotFound(err) { - // Nothing to do as it does not exists + // We cannot do further cleanup of the MariaDBDatabase and + // MariaDBAccount as their name is only available in the NovaCell CR + // since the cell definition is removed from the Nova CR already. return nil } if err != nil { @@ -726,17 +689,12 @@ func (r *NovaReconciler) ensureCellDeleted( } // If it is not created by us, we don't touch it if !OwnedBy(cell, instance) { - Log.Info("CellName isn't defined in the nova, but there is a "+ + Log.Info("Cell isn't defined in the Nova, but there is a "+ "Cell CR not owned by us. Not deleting it.", "cell", cell) return nil } - err = r.Client.Delete(ctx, cell) - if err != nil && k8s_errors.IsNotFound(err) { - return err - } - dbName, accountName := novaapi.ServiceName+"-"+cell.Spec.CellName, cell.Spec.CellDatabaseAccount configHash, scriptName, configName, err := r.ensureNovaManageJobSecret(ctx, h, instance, @@ -765,7 +723,16 @@ func (r *NovaReconciler) ensureCellDeleted( secretName := getNovaCellCRName(instance.Name, cellName) err = secret.DeleteSecretsWithName(ctx, h, secretName, instance.Namespace) - if err != nil { + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + configSecret, scriptSecret := r.getNovaManageJobSecretNames(cell) + err = secret.DeleteSecretsWithName(ctx, h, configSecret, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + err = secret.DeleteSecretsWithName(ctx, h, scriptSecret, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { return err } @@ -777,17 +744,42 @@ func (r *NovaReconciler) ensureCellDeleted( }, } err = r.Client.Delete(ctx, transportURL) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + err = mariadbv1.DeleteDatabaseAndAccountFinalizers( + ctx, h, dbName, accountName, instance.Namespace) if err != nil { return err } - err = DeleteDatabaseAndAccountFinalizers(ctx, h, dbName, accountName, instance.ObjectMeta.Namespace) - + err = r.ensureAccountDeletedIfOwned(ctx, h, instance, accountName) if err != nil { return err } - Log.Info("Cell isn't defined in the nova, so deleted cell", "cell", cell) + + database := &mariadbv1.MariaDBDatabase{ + ObjectMeta: metav1.ObjectMeta{ + Name: dbName, + Namespace: instance.Namespace, + }, + } + err = r.Client.Delete(ctx, database) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + Log.Info("Deleted MariaDBDatabase", "database", database) + + // Finally we delete the NovaCell CR. We need to do it as the last step + // otherwise we cannot retry the above cleanup as we won't have the data + // what to clean up. + err = r.Client.Delete(ctx, cell) + if err != nil && k8s_errors.IsNotFound(err) { + return err + } + + Log.Info("Cell isn't defined in the Nova CR, so it is deleted", "cell", cell) return nil } @@ -899,6 +891,14 @@ func (r *NovaReconciler) initConditions( return nil } +func (r *NovaReconciler) getNovaManageJobSecretNames( + cell *novav1.NovaCell, +) (configName string, scriptName string) { + configName = fmt.Sprintf("%s-config-data", cell.Name+"-manage") + scriptName = fmt.Sprintf("%s-scripts", cell.Name+"-manage") + return +} + func (r *NovaReconciler) ensureNovaManageJobSecret( ctx context.Context, h *helper.Helper, instance *novav1.Nova, cell *novav1.NovaCell, @@ -907,8 +907,7 @@ func (r *NovaReconciler) ensureNovaManageJobSecret( cellTransportURL string, cellDB *mariadbv1.Database, ) (map[string]env.Setter, string, string, error) { - configName := fmt.Sprintf("%s-config-data", cell.Name+"-manage") - scriptName := fmt.Sprintf("%s-scripts", cell.Name+"-manage") + configName, scriptName := r.getNovaManageJobSecretNames(cell) cmLabels := labels.GetLabels( instance, labels.GetGroupLabel(NovaLabelPrefix), map[string]string{}, diff --git a/go.mod b/go.mod index e57b7d16e..0c20bc1f2 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/openstack-k8s-operators/lib-common/modules/common v0.4.1-0.20240926101719-8fc1c3da53f7 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.4.1-0.20240926101719-8fc1c3da53f7 github.com/openstack-k8s-operators/lib-common/modules/test v0.4.1-0.20240926101719-8fc1c3da53f7 - github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20240927143624-61d230f582d6 + github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20241008102252-6f4b7a67029f github.com/openstack-k8s-operators/nova-operator/api v0.0.0-20221209164002-f9e6b9363961 go.uber.org/zap v1.27.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index b5484b430..e4b3a193a 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/openstack-k8s-operators/lib-common/modules/openstack v0.4.1-0.2024092 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.4.1-0.20240926101719-8fc1c3da53f7/go.mod h1:oCopeVBDHbCoPFXOMNAPae/XvPQ3H9Sdaag16HlLZC0= github.com/openstack-k8s-operators/lib-common/modules/test v0.4.1-0.20240926101719-8fc1c3da53f7 h1:dNf8GQ+6Dv2twGwYlyTfiYttpSESfcQ89p0py2XAbdo= github.com/openstack-k8s-operators/lib-common/modules/test v0.4.1-0.20240926101719-8fc1c3da53f7/go.mod h1:LV0jo5etIsGyINpmB37i4oWR8zU6ApIuh7fsqGGA41o= -github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20240927143624-61d230f582d6 h1:t0TJCqVvIR+2ab77er24bjUTS8uR2xJuuXt3cBlom90= -github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20240927143624-61d230f582d6/go.mod h1:13K91HQShjM0y1zVTupCybaTpWJYzOhMPd+rJUpxIg8= +github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20241008102252-6f4b7a67029f h1:Tkg3SwrWfLhlc00MQYYLli5TdUQCxC5KFDq5BToSVaw= +github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20241008102252-6f4b7a67029f/go.mod h1:13K91HQShjM0y1zVTupCybaTpWJYzOhMPd+rJUpxIg8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/test/functional/nova_reconfiguration_test.go b/test/functional/nova_reconfiguration_test.go index 11c5b44bf..a7f5b1094 100644 --- a/test/functional/nova_reconfiguration_test.go +++ b/test/functional/nova_reconfiguration_test.go @@ -69,8 +69,10 @@ func CreateNovaWith3CellsAndEnsureReady(novaNames NovaNames) { cell1Account, cell1Secret := mariadb.CreateMariaDBAccountAndSecret( cell1.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{}) - DeferCleanup(k8sClient.Delete, ctx, cell1Account) - DeferCleanup(k8sClient.Delete, ctx, cell1Secret) + DeferCleanup(th.DeleteInstance, cell1Account) + DeferCleanup( + th.DeleteSecret, + types.NamespacedName{Name: cell1Secret.Name, Namespace: cell1Secret.Namespace}) cell2Account, cell2Secret := mariadb.CreateMariaDBAccountAndSecret( cell2.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{}) From 37f25bd01a9da9800c9aad00f851587f60e45505 Mon Sep 17 00:00:00 2001 From: Balazs Gibizer Date: Mon, 7 Oct 2024 14:34:27 +0200 Subject: [PATCH 9/9] [kuttl]Assert DB and Secrets are deleted for deleted cell --- .../default/cell-tests/02-assert.yaml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/kuttl/test-suites/default/cell-tests/02-assert.yaml b/test/kuttl/test-suites/default/cell-tests/02-assert.yaml index 807eefe98..c5a5f9ed0 100644 --- a/test/kuttl/test-suites/default/cell-tests/02-assert.yaml +++ b/test/kuttl/test-suites/default/cell-tests/02-assert.yaml @@ -126,3 +126,32 @@ commands: - script: | oc get -n nova-kuttl-default pod/nova-kuttl-cell1-conductor-0 || exit 0 exit 1 + - script: | + oc get -n nova-kuttl-default NovaCell/nova-cell1 || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default MariaDBAccount/nova-cell1 || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default MariaDBDatabase/nova-cell1 || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default TransportUrl/nova-cell1-transport || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default Secret/nova-cell1 || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default Secret/nova-cell1-db-secret || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default Secret/nova-cell1-compute-config || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default Secret/nova-cell1-manage-config-data || exit 0 + exit 1 + # TODO(gibi): this needs an openstack-operator change to clean up the cell1 + # cert secrets + #- script: | + # oc get -n nova-kuttl-default Secret/cert-nova-novncproxy-cell1-public-svc || exit 0 + # exit 1