diff --git a/controllers/nova_controller.go b/controllers/nova_controller.go index dd0844f79..096eb25af 100644 --- a/controllers/nova_controller.go +++ b/controllers/nova_controller.go @@ -591,10 +591,198 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul } } + // We need to check and delete cells + novaCellList := &novav1.NovaCellList{} + listOpts := []client.ListOption{ + client.InNamespace(instance.Namespace), + } + if err := r.Client.List(ctx, novaCellList, listOpts...); err != nil { + return ctrl.Result{}, err + } + + for _, cr := range novaCellList.Items { + _, ok := instance.Spec.CellTemplates[cr.Spec.CellName] + if !ok { + err := r.ensureCellDeleted(ctx, h, instance, + cr.Spec.CellName, apiTransportURL, + secret, apiDB, cellDBs[novav1.Cell0Name].Database.GetDatabaseHostname(), cells[novav1.Cell0Name]) + if err != nil { + return ctrl.Result{}, err + } + Log.Info("Cell deleted", "cell", cr.Spec.CellName) + delete(instance.Status.RegisteredCells, cr.Name) + } + + } + Log.Info("Successfully reconciled") return ctrl.Result{}, nil } +func (r *NovaReconciler) ensureAccountDeletedIfOwned( + ctx context.Context, + h *helper.Helper, + instance *novav1.Nova, + accountName string, +) error { + Log := r.GetLogger(ctx) + + account, err := mariadbv1.GetAccount(ctx, h, accountName, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + if k8s_errors.IsNotFound(err) { + // Nothing to delete + return nil + } + + // If it is not created by us, we don't clean it up + if !OwnedBy(account, instance) { + Log.Info("MariaDBAccount in not owned by Nova, not deleting", "account", account) + return nil + } + + // NOTE(gibi): We need to delete the Secret first and then the Account + // otherwise we cannot retry the Secret deletion when the Account is + // gone as we will not know the name of the Secret. This logic should + // be moved to the mariadb-operator. + err = secret.DeleteSecretsWithName(ctx, h, account.Spec.Secret, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + err = r.Client.Delete(ctx, account) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + + Log.Info("Deleted MariaDBAccount", "account", account) + return nil +} + +func (r *NovaReconciler) ensureCellDeleted( + ctx context.Context, + h *helper.Helper, + instance *novav1.Nova, + cellName string, + apiTransportURL string, + topLevelSecret corev1.Secret, + apiDB *mariadbv1.Database, + APIDatabaseHostname string, + cell0 *novav1.NovaCell, +) error { + Log := r.GetLogger(ctx) + cell := &novav1.NovaCell{} + fullCellName := types.NamespacedName{ + Name: getNovaCellCRName(instance.Name, cellName), + Namespace: instance.GetNamespace(), + } + + err := r.Client.Get(ctx, fullCellName, cell) + if k8s_errors.IsNotFound(err) { + // We cannot do further cleanup of the MariaDBDatabase and + // MariaDBAccount as their name is only available in the NovaCell CR + // since the cell definition is removed from the Nova CR already. + return nil + } + if err != nil { + return err + } + // If it is not created by us, we don't touch it + if !OwnedBy(cell, instance) { + Log.Info("Cell isn't defined in the Nova, but there is a "+ + "Cell CR not owned by us. Not deleting it.", + "cell", cell) + return nil + } + + dbName, accountName := novaapi.ServiceName+"-"+cell.Spec.CellName, cell.Spec.CellDatabaseAccount + + configHash, scriptName, configName, err := r.ensureNovaManageJobSecret(ctx, h, instance, + cell0, topLevelSecret, APIDatabaseHostname, apiTransportURL, apiDB) + if err != nil { + return err + } + inputHash, err := util.HashOfInputHashes(configHash) + if err != nil { + return err + } + + labels := map[string]string{ + common.AppSelector: NovaLabelPrefix, + } + jobDef := nova.CellDeleteJob(instance, cell, configName, scriptName, inputHash, labels) + job := job.NewJob( + jobDef, cell.Name+"-cell-delete", + instance.Spec.PreserveJobs, r.RequeueTimeout, + inputHash) + + _, err = job.DoJob(ctx, h) + if err != nil { + return err + } + + secretName := getNovaCellCRName(instance.Name, cellName) + err = secret.DeleteSecretsWithName(ctx, h, secretName, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + configSecret, scriptSecret := r.getNovaManageJobSecretNames(cell) + err = secret.DeleteSecretsWithName(ctx, h, configSecret, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + err = secret.DeleteSecretsWithName(ctx, h, scriptSecret, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + + // Delete transportURL cr + transportURL := &rabbitmqv1.TransportURL{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name + "-" + cellName + "-transport", + Namespace: instance.Namespace, + }, + } + err = r.Client.Delete(ctx, transportURL) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + + err = mariadbv1.DeleteDatabaseAndAccountFinalizers( + ctx, h, dbName, accountName, instance.Namespace) + if err != nil { + return err + } + + err = r.ensureAccountDeletedIfOwned(ctx, h, instance, accountName) + if err != nil { + return err + } + + database := &mariadbv1.MariaDBDatabase{ + ObjectMeta: metav1.ObjectMeta{ + Name: dbName, + Namespace: instance.Namespace, + }, + } + err = r.Client.Delete(ctx, database) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + Log.Info("Deleted MariaDBDatabase", "database", database) + + // Finally we delete the NovaCell CR. We need to do it as the last step + // otherwise we cannot retry the above cleanup as we won't have the data + // what to clean up. + err = r.Client.Delete(ctx, cell) + if err != nil && k8s_errors.IsNotFound(err) { + return err + } + + Log.Info("Cell isn't defined in the Nova CR, so it is deleted", "cell", cell) + return nil +} + func (r *NovaReconciler) initStatus( instance *novav1.Nova, ) error { @@ -703,6 +891,14 @@ func (r *NovaReconciler) initConditions( return nil } +func (r *NovaReconciler) getNovaManageJobSecretNames( + cell *novav1.NovaCell, +) (configName string, scriptName string) { + configName = fmt.Sprintf("%s-config-data", cell.Name+"-manage") + scriptName = fmt.Sprintf("%s-scripts", cell.Name+"-manage") + return +} + func (r *NovaReconciler) ensureNovaManageJobSecret( ctx context.Context, h *helper.Helper, instance *novav1.Nova, cell *novav1.NovaCell, @@ -711,8 +907,7 @@ func (r *NovaReconciler) ensureNovaManageJobSecret( cellTransportURL string, cellDB *mariadbv1.Database, ) (map[string]env.Setter, string, string, error) { - configName := fmt.Sprintf("%s-config-data", cell.Name+"-manage") - scriptName := fmt.Sprintf("%s-scripts", cell.Name+"-manage") + configName, scriptName := r.getNovaManageJobSecretNames(cell) cmLabels := labels.GetLabels( instance, labels.GetGroupLabel(NovaLabelPrefix), map[string]string{}, diff --git a/go.mod b/go.mod index e57b7d16e..0c20bc1f2 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/openstack-k8s-operators/lib-common/modules/common v0.4.1-0.20240926101719-8fc1c3da53f7 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.4.1-0.20240926101719-8fc1c3da53f7 github.com/openstack-k8s-operators/lib-common/modules/test v0.4.1-0.20240926101719-8fc1c3da53f7 - github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20240927143624-61d230f582d6 + github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20241008102252-6f4b7a67029f github.com/openstack-k8s-operators/nova-operator/api v0.0.0-20221209164002-f9e6b9363961 go.uber.org/zap v1.27.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index b5484b430..e4b3a193a 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/openstack-k8s-operators/lib-common/modules/openstack v0.4.1-0.2024092 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.4.1-0.20240926101719-8fc1c3da53f7/go.mod h1:oCopeVBDHbCoPFXOMNAPae/XvPQ3H9Sdaag16HlLZC0= github.com/openstack-k8s-operators/lib-common/modules/test v0.4.1-0.20240926101719-8fc1c3da53f7 h1:dNf8GQ+6Dv2twGwYlyTfiYttpSESfcQ89p0py2XAbdo= github.com/openstack-k8s-operators/lib-common/modules/test v0.4.1-0.20240926101719-8fc1c3da53f7/go.mod h1:LV0jo5etIsGyINpmB37i4oWR8zU6ApIuh7fsqGGA41o= -github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20240927143624-61d230f582d6 h1:t0TJCqVvIR+2ab77er24bjUTS8uR2xJuuXt3cBlom90= -github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20240927143624-61d230f582d6/go.mod h1:13K91HQShjM0y1zVTupCybaTpWJYzOhMPd+rJUpxIg8= +github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20241008102252-6f4b7a67029f h1:Tkg3SwrWfLhlc00MQYYLli5TdUQCxC5KFDq5BToSVaw= +github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20241008102252-6f4b7a67029f/go.mod h1:13K91HQShjM0y1zVTupCybaTpWJYzOhMPd+rJUpxIg8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/pkg/nova/celldelete.go b/pkg/nova/celldelete.go new file mode 100644 index 000000000..9ee45b727 --- /dev/null +++ b/pkg/nova/celldelete.go @@ -0,0 +1,86 @@ +package nova + +import ( + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" +) + +func CellDeleteJob( + instance *novav1.Nova, + cell *novav1.NovaCell, + configName string, + scriptName string, + inputHash string, + labels map[string]string, +) *batchv1.Job { + args := []string{"-c", KollaServiceCommand} + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["KOLLA_BOOTSTRAP"] = env.SetValue("true") + envVars["CELL_NAME"] = env.SetValue(cell.Spec.CellName) + + // This is stored in the Job so that if the input of the job changes + // then it results in a new job hash and therefore lib-common will re-run + // the job + envVars["INPUT_HASH"] = env.SetValue(inputHash) + + env := env.MergeEnvs([]corev1.EnvVar{}, envVars) + + jobName := instance.Name + "-" + cell.Spec.CellName + "-cell-delete" + + volumes := []corev1.Volume{ + GetConfigVolume(configName), + GetScriptVolume(scriptName), + } + volumeMounts := []corev1.VolumeMount{ + GetConfigVolumeMount(), + GetScriptVolumeMount(), + GetKollaConfigVolumeMount("cell-delete"), + } + + // add CA cert if defined + if instance.Spec.APIServiceTemplate.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.APIServiceTemplate.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.APIServiceTemplate.TLS.CreateVolumeMounts(nil)...) + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + Volumes: volumes, + Containers: []corev1.Container{ + { + Name: "nova-manage", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: cell.Spec.ConductorContainerImageURL, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(NovaUserID), + }, + Env: env, + VolumeMounts: volumeMounts, + }, + }, + }, + }, + }, + } + + return job +} diff --git a/templates/nova-manage/bin/delete_cell.sh b/templates/nova-manage/bin/delete_cell.sh new file mode 100755 index 000000000..21ed94591 --- /dev/null +++ b/templates/nova-manage/bin/delete_cell.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2023. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +export CELL_NAME=${CELL_NAME:?"Please specify a CELL_NAME variable."} + +# NOTE(gibi): nova-manage should be enhanced upstream to get rid of this +# uglyness +# Note the "|" around the CELL_NAME, that is needed as a single line from +# nova-manage cell_v2 cell_list can match to multiple cells if the cell name +# is part of the line, e.g. as the user name of the DB URL +cell_uuid=$(nova-manage cell_v2 list_cells | tr ' ' '|' | tr --squeeze-repeats '|' | grep -e "^|$CELL_NAME|" | cut -d '|' -f 3) + +nova-manage cell_v2 delete_cell --cell_uuid "${cell_uuid}" diff --git a/templates/nova-manage/config/cell-delete-config.json b/templates/nova-manage/config/cell-delete-config.json new file mode 100644 index 000000000..955ea4ca2 --- /dev/null +++ b/templates/nova-manage/config/cell-delete-config.json @@ -0,0 +1,36 @@ +{ + "command": "/bin/delete_cell.sh", + "config_files": [ + { + "source": "/var/lib/openstack/config/nova-blank.conf", + "dest": "/etc/nova/nova.conf", + "owner": "nova", + "perm": "0600" + }, + { + "source": "/var/lib/openstack/config/01-nova.conf", + "dest": "/etc/nova/nova.conf.d/01-nova.conf", + "owner": "nova", + "perm": "0600" + }, + { + "source": "/var/lib/openstack/config/02-nova-override.conf", + "dest": "/etc/nova/nova.conf.d/02-nova-override.conf", + "owner": "nova", + "perm": "0600", + "optional": true + }, + { + "source": "/var/lib/openstack/bin/delete_cell.sh", + "dest": "/bin/", + "owner": "nova", + "perm": "0700" + }, + { + "source": "/var/lib/openstack/config/my.cnf", + "dest": "/etc/my.cnf", + "owner": "nova", + "perm": "0644" + } + ] +} diff --git a/test/functional/base_test.go b/test/functional/base_test.go index d5e493666..34624170c 100644 --- a/test/functional/base_test.go +++ b/test/functional/base_test.go @@ -350,6 +350,7 @@ type CellNames struct { ConductorStatefulSetName types.NamespacedName TransportURLName types.NamespacedName CellMappingJobName types.NamespacedName + CellDeleteJobName types.NamespacedName MetadataName types.NamespacedName MetadataStatefulSetName types.NamespacedName MetadataConfigDataName types.NamespacedName @@ -418,6 +419,10 @@ func GetCellNames(novaName types.NamespacedName, cell string) CellNames { Namespace: novaName.Namespace, Name: cellName.Name + "-cell-mapping", }, + CellDeleteJobName: types.NamespacedName{ + Namespace: novaName.Namespace, + Name: cellName.Name + "-cell-delete", + }, ConductorConfigDataName: types.NamespacedName{ Namespace: novaName.Namespace, Name: cellConductor.Name + "-config-data", diff --git a/test/functional/nova_reconfiguration_test.go b/test/functional/nova_reconfiguration_test.go index 89db43011..a7f5b1094 100644 --- a/test/functional/nova_reconfiguration_test.go +++ b/test/functional/nova_reconfiguration_test.go @@ -22,11 +22,13 @@ import ( "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports . "github.com/onsi/gomega" //revive:disable:dot-imports + k8s_errors "k8s.io/apimachinery/pkg/api/errors" //revive:disable-next-line:dot-imports . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -67,8 +69,10 @@ func CreateNovaWith3CellsAndEnsureReady(novaNames NovaNames) { cell1Account, cell1Secret := mariadb.CreateMariaDBAccountAndSecret( cell1.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{}) - DeferCleanup(k8sClient.Delete, ctx, cell1Account) - DeferCleanup(k8sClient.Delete, ctx, cell1Secret) + DeferCleanup(th.DeleteInstance, cell1Account) + DeferCleanup( + th.DeleteSecret, + types.NamespacedName{Name: cell1Secret.Name, Namespace: cell1Secret.Namespace}) cell2Account, cell2Secret := mariadb.CreateMariaDBAccountAndSecret( cell2.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{}) @@ -168,6 +172,37 @@ var _ = Describe("Nova reconfiguration", func() { CreateNovaWith3CellsAndEnsureReady(novaNames) }) + When("cell1 is deleted", func() { + It("cell cr is deleted", func() { + Eventually(func(g Gomega) { + nova := GetNova(novaNames.NovaName) + + delete(nova.Spec.CellTemplates, "cell1") + + g.Expect(k8sClient.Update(ctx, nova)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + nova := GetNova(novaNames.NovaName) + g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell1.CellCRName.Name)) + }, timeout, interval).Should(Succeed()) + + NovaCellNotExists(cell1.CellCRName) + Eventually(func(g Gomega) { + mappingJob := th.GetJob(cell1.CellDeleteJobName) + newJobInputHash := GetEnvVarValue( + mappingJob.Spec.Template.Spec.Containers[0].Env, "INPUT_HASH", "") + g.Expect(newJobInputHash).NotTo(BeNil()) + }, timeout, interval).Should(Succeed()) + th.AssertSecretDoesNotExist(cell1.InternalCellSecretName) + + Eventually(func(g Gomega) { + instance := &rabbitmqv1.TransportURL{} + err := k8sClient.Get(ctx, cell1.TransportURLName, instance) + g.Expect(k8s_errors.IsNotFound(err)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + }) When("cell0 conductor replicas is set to 0", func() { It("sets the deployment replicas to 0", func() { cell0DeploymentName := cell0.ConductorStatefulSetName diff --git a/test/kuttl/test-suites/default/cell-tests/00-cleanup-nova.yaml b/test/kuttl/test-suites/default/cell-tests/00-cleanup-nova.yaml new file mode 120000 index 000000000..23d51de43 --- /dev/null +++ b/test/kuttl/test-suites/default/cell-tests/00-cleanup-nova.yaml @@ -0,0 +1 @@ +../common/cleanup-nova.yaml \ No newline at end of file diff --git a/test/kuttl/test-suites/default/cell-tests/01-assert.yaml b/test/kuttl/test-suites/default/cell-tests/01-assert.yaml new file mode 100644 index 000000000..54a7bc6ca --- /dev/null +++ b/test/kuttl/test-suites/default/cell-tests/01-assert.yaml @@ -0,0 +1,623 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: Nova +metadata: + finalizers: + - openstack.org/nova + name: nova-kuttl + namespace: nova-kuttl-default +spec: + apiDatabaseInstance: openstack + apiDatabaseAccount: nova-api + apiMessageBusInstance: rabbitmq + apiServiceTemplate: + customServiceConfig: "" + replicas: 1 + resources: {} + cellTemplates: + cell0: + cellDatabaseInstance: openstack + cellDatabaseAccount: nova-cell0 + cellMessageBusInstance: rabbitmq + conductorServiceTemplate: + customServiceConfig: "" + replicas: 1 + resources: {} + hasAPIAccess: true + metadataServiceTemplate: + enabled: false + noVNCProxyServiceTemplate: + enabled: false + cell1: + cellDatabaseInstance: openstack-cell1 + cellDatabaseAccount: nova-cell1 + cellMessageBusInstance: rabbitmq-cell1 + conductorServiceTemplate: + customServiceConfig: "" + replicas: 1 + resources: {} + hasAPIAccess: true + metadataServiceTemplate: + enabled: false + noVNCProxyServiceTemplate: + enabled: true + customServiceConfig: "" + replicas: 1 + resources: {} + keystoneInstance: keystone + metadataServiceTemplate: + enabled: true + customServiceConfig: "" + replicas: 1 + resources: {} + passwordSelectors: + metadataSecret: MetadataSecret + service: NovaPassword + schedulerServiceTemplate: + customServiceConfig: "" + replicas: 1 + resources: {} + secret: osp-secret + serviceUser: nova +status: + apiServiceReadyCount: 1 + metadataServiceReadyCount: 1 + schedulerServiceReadyCount: 1 + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: Setup complete + reason: Ready + status: "True" + type: KeystoneServiceReady + - message: MariaDBAccount creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady + - message: ' Memcached instance has been provisioned' + reason: Ready + status: "True" + type: MemcachedReady + - message: DB create completed + reason: Ready + status: "True" + type: NovaAPIDBReady + - message: API message bus creation successfully + reason: Ready + status: "True" + type: NovaAPIMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaAPIReady + - message: All DBs created successfully + reason: Ready + status: "True" + type: NovaAllCellDBReady + - message: All NovaCells are ready + reason: Ready + status: "True" + type: NovaAllCellReady + - message: All message busses created successfully + reason: Ready + status: "True" + type: NovaAllCellsMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaMetadataReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaSchedulerReady + - message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - message: Role created + reason: Ready + status: "True" + type: RoleReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-api + name: nova-kuttl-api-0 +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified + name: nova-kuttl-api-api + ready: true + started: true + - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified + name: nova-kuttl-api-log + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + service: nova-scheduler + name: nova-kuttl-scheduler-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-scheduler +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-scheduler:current-podified + name: nova-kuttl-scheduler-scheduler + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-metadata + statefulset.kubernetes.io/pod-name: nova-kuttl-metadata-0 + name: nova-kuttl-metadata-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-metadata +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified + name: nova-kuttl-metadata-log + ready: true + started: true + - image: quay.io/podified-antelope-centos9/openstack-nova-api:current-podified + name: nova-kuttl-metadata-metadata + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + cell: cell0 + service: nova-conductor + statefulset.kubernetes.io/pod-name: nova-kuttl-cell0-conductor-0 + name: nova-kuttl-cell0-conductor-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-cell0-conductor +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified + name: nova-kuttl-cell0-conductor-conductor + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + cell: cell1 + service: nova-conductor + statefulset.kubernetes.io/pod-name: nova-kuttl-cell1-conductor-0 + name: nova-kuttl-cell1-conductor-0 + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-cell1-conductor +status: + containerStatuses: + - image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified + name: nova-kuttl-cell1-conductor-conductor + ready: true + started: true +--- +apiVersion: v1 +kind: Service +metadata: + labels: + endpoint: internal + service: nova-api + name: nova-internal + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaAPI + name: nova-kuttl-api +spec: + ports: + - name: nova-internal + selector: + service: nova-api +--- +apiVersion: v1 +kind: Service +metadata: + labels: + endpoint: public + service: nova-api + name: nova-public + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaAPI + name: nova-kuttl-api +spec: + ports: + - name: nova-public + selector: + service: nova-api +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-api.openstack.org/name: nova-kuttl-api + name: nova-kuttl-api-config-data + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaAPI + name: nova-kuttl-api +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-conductor.openstack.org/name: nova-kuttl-cell0-conductor + name: nova-kuttl-cell0-conductor-config-data + namespace: nova-kuttl-default + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell0-conductor +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-conductor.openstack.org/name: nova-kuttl-cell0-conductor + name: nova-kuttl-cell0-conductor-scripts + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell0-conductor +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-conductor.openstack.org/name: nova-kuttl-cell1-conductor + name: nova-kuttl-cell1-conductor-config-data + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell1-conductor +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-conductor.openstack.org/name: nova-kuttl-cell1-conductor + name: nova-kuttl-cell1-conductor-scripts + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell1-conductor +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-metadata.openstack.org/name: nova-kuttl-metadata + name: nova-kuttl-metadata-config-data + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaMetadata + name: nova-kuttl-metadata +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-scheduler.openstack.org/name: nova-kuttl-scheduler + name: nova-kuttl-scheduler-config-data + ownerReferences: + - blockOwnerDeletion: true + controller: true + kind: NovaScheduler + name: nova-kuttl-scheduler +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + nova-cell.openstack.org/name: nova-kuttl-cell1 + name: nova-kuttl-cell1-compute-config + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaCell + name: nova-kuttl-cell1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-api + namespace: nova-kuttl-default + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaAPI + name: nova-kuttl-api +spec: + replicas: 1 + selector: + matchLabels: + service: nova-api + serviceName: "" + template: + metadata: + labels: + service: nova-api +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-metadata + namespace: nova-kuttl-default + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaMetadata + name: nova-kuttl-metadata +spec: + replicas: 1 + selector: + matchLabels: + service: nova-metadata + template: + metadata: + labels: + service: nova-metadata +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-scheduler + namespace: nova-kuttl-default + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaScheduler + name: nova-kuttl-scheduler +spec: + replicas: 1 + selector: + matchLabels: + service: nova-scheduler + template: + metadata: + labels: + service: nova-scheduler +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-cell0-conductor + namespace: nova-kuttl-default + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell0-conductor +spec: + replicas: 1 + selector: + matchLabels: + cell: cell0 + service: nova-conductor + template: + metadata: + labels: + cell: cell0 + service: nova-conductor +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nova-kuttl-cell1-conductor + namespace: nova-kuttl-default + ownerReferences: + - apiVersion: nova.openstack.org/v1beta1 + blockOwnerDeletion: true + controller: true + kind: NovaConductor + name: nova-kuttl-cell1-conductor +spec: + replicas: 1 + selector: + matchLabels: + cell: cell1 + service: nova-conductor + template: + metadata: + labels: + cell: cell1 + service: nova-conductor +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + labels: + service: nova-conductor + name: nova-kuttl-cell0-db-purge + namespace: nova-kuttl-default +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + completions: 1 + parallelism: 1 + template: + spec: + containers: + - args: + - -c + - /usr/local/bin/kolla_start + command: + - /bin/bash + env: + - name: ARCHIVE_AGE + value: "30" + - name: KOLLA_BOOTSTRAP + value: "true" + - name: KOLLA_CONFIG_STRATEGY + value: COPY_ALWAYS + - name: PURGE_AGE + value: "90" + image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified + name: nova-manage + securityContext: + runAsUser: 42436 + volumeMounts: + - mountPath: /var/lib/openstack/config + name: config-data + - mountPath: /var/lib/openstack/bin + name: scripts + - mountPath: /var/lib/kolla/config_files/config.json + name: config-data + subPath: nova-conductor-dbpurge-config.json + restartPolicy: OnFailure + serviceAccount: nova-nova-kuttl + serviceAccountName: nova-nova-kuttl + volumes: + - name: config-data + secret: + secretName: nova-kuttl-cell0-conductor-config-data + - name: scripts + secret: + secretName: nova-kuttl-cell0-conductor-scripts + schedule: 0 0 * * * + successfulJobsHistoryLimit: 3 + suspend: false +status: {} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + labels: + service: nova-conductor + name: nova-kuttl-cell1-db-purge + namespace: nova-kuttl-default +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + completions: 1 + parallelism: 1 + template: + spec: + containers: + - args: + - -c + - /usr/local/bin/kolla_start + command: + - /bin/bash + env: + - name: ARCHIVE_AGE + value: "30" + - name: KOLLA_BOOTSTRAP + value: "true" + - name: KOLLA_CONFIG_STRATEGY + value: COPY_ALWAYS + - name: PURGE_AGE + value: "90" + image: quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified + name: nova-manage + securityContext: + runAsUser: 42436 + volumeMounts: + - mountPath: /var/lib/openstack/config + name: config-data + - mountPath: /var/lib/openstack/bin + name: scripts + - mountPath: /var/lib/kolla/config_files/config.json + name: config-data + subPath: nova-conductor-dbpurge-config.json + restartPolicy: OnFailure + serviceAccount: nova-nova-kuttl + serviceAccountName: nova-nova-kuttl + volumes: + - name: config-data + secret: + secretName: nova-kuttl-cell1-conductor-config-data + - name: scripts + secret: + secretName: nova-kuttl-cell1-conductor-scripts + schedule: 0 0 * * * + successfulJobsHistoryLimit: 3 + suspend: false +status: {} diff --git a/test/kuttl/test-suites/default/cell-tests/01-deploy.yaml b/test/kuttl/test-suites/default/cell-tests/01-deploy.yaml new file mode 100644 index 000000000..daae034aa --- /dev/null +++ b/test/kuttl/test-suites/default/cell-tests/01-deploy.yaml @@ -0,0 +1,6 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: Nova +metadata: + name: nova-kuttl +spec: + secret: osp-secret diff --git a/test/kuttl/test-suites/default/cell-tests/02-assert.yaml b/test/kuttl/test-suites/default/cell-tests/02-assert.yaml new file mode 100644 index 000000000..c5a5f9ed0 --- /dev/null +++ b/test/kuttl/test-suites/default/cell-tests/02-assert.yaml @@ -0,0 +1,157 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: Nova +metadata: + finalizers: + - openstack.org/nova + name: nova-kuttl + namespace: nova-kuttl-default +status: + apiServiceReadyCount: 1 + metadataServiceReadyCount: 1 + schedulerServiceReadyCount: 1 + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: Setup complete + reason: Ready + status: "True" + type: KeystoneServiceReady + - message: MariaDBAccount creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady + - message: ' Memcached instance has been provisioned' + reason: Ready + status: "True" + type: MemcachedReady + - message: DB create completed + reason: Ready + status: "True" + type: NovaAPIDBReady + - message: API message bus creation successfully + reason: Ready + status: "True" + type: NovaAPIMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaAPIReady + - message: All DBs created successfully + reason: Ready + status: "True" + type: NovaAllCellDBReady + - message: All NovaCells are ready + reason: Ready + status: "True" + type: NovaAllCellReady + - message: All message busses created successfully + reason: Ready + status: "True" + type: NovaAllCellsMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaMetadataReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaSchedulerReady + - message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - message: Role created + reason: Ready + status: "True" + type: RoleReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-api + name: nova-kuttl-api-0 +status: + containerStatuses: + - name: nova-kuttl-api-api + ready: true + started: true + - name: nova-kuttl-api-log + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-metadata + statefulset.kubernetes.io/pod-name: nova-kuttl-metadata-0 + name: nova-kuttl-metadata-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-metadata +status: + containerStatuses: + - name: nova-kuttl-metadata-log + ready: true + started: true + - name: nova-kuttl-metadata-metadata + ready: true + started: true + +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + oc get -n nova-kuttl-default pod/nova-kuttl-cell1-novncproxy-0 || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default pod/nova-kuttl-cell1-conductor-0 || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default NovaCell/nova-cell1 || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default MariaDBAccount/nova-cell1 || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default MariaDBDatabase/nova-cell1 || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default TransportUrl/nova-cell1-transport || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default Secret/nova-cell1 || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default Secret/nova-cell1-db-secret || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default Secret/nova-cell1-compute-config || exit 0 + exit 1 + - script: | + oc get -n nova-kuttl-default Secret/nova-cell1-manage-config-data || exit 0 + exit 1 + # TODO(gibi): this needs an openstack-operator change to clean up the cell1 + # cert secrets + #- script: | + # oc get -n nova-kuttl-default Secret/cert-nova-novncproxy-cell1-public-svc || exit 0 + # exit 1 diff --git a/test/kuttl/test-suites/default/cell-tests/02-delete-cell-nova.yaml b/test/kuttl/test-suites/default/cell-tests/02-delete-cell-nova.yaml new file mode 100644 index 000000000..df6625a64 --- /dev/null +++ b/test/kuttl/test-suites/default/cell-tests/02-delete-cell-nova.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc patch -n nova-kuttl-default nova/nova-kuttl --type='json' -p='[{"op": "remove", "path": "/spec/cellTemplates/cell1"}]' diff --git a/test/kuttl/test-suites/default/cell-tests/03-cleanup-nova.yaml b/test/kuttl/test-suites/default/cell-tests/03-cleanup-nova.yaml new file mode 120000 index 000000000..49b58eee9 --- /dev/null +++ b/test/kuttl/test-suites/default/cell-tests/03-cleanup-nova.yaml @@ -0,0 +1 @@ +00-cleanup-nova.yaml \ No newline at end of file