From c08080c53c2cbe569a30eeb96e788995bdeee307 Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Wed, 11 Oct 2023 10:41:52 +0200 Subject: [PATCH] Enable pruner and cleaner CronJobs This patch enables both pruner and cleaner cronJobs that are supposed to manage the /var/lib/glance/image-cache directory to stay under the upper limit defined by ImageCacheSize (that triggers a PVC request of that size), as well as clean the incomplete/stalled images from the related folders. In order to make the above happen, a few things are fixed by this patch: 1. 00-config.conf (the global one) is updated to include the image-cache related parameters (if cache is enabled); 2. the CronJobs creation logic has been moved to a dedicated 'ensureCronJobs' function (it's the same approach we adopted for PVC creation) and results cleaner from the glance_controller point of view; 3. The CronJob function itself mounts an additional Volume in case of cleaner/pruner: the glance-cache PVC should be mounted in /var/lib/glance/image-cache because it represents the target directory managed by the cronJob command; in case of DBPurge this additional volume is not provided 4. The 'image_cache_max_size' parameter was broken and filled with a wrong value, hence with this patch we normalize the ImageCacheSize parameter (resource.MustParse()) and pass the resulting Value() to the templateParameters that are supposed to fill the right section of the glance config file Signed-off-by: Francesco Pantano --- controllers/glance_controller.go | 70 +++++++++++++++++++++++++---- controllers/glanceapi_controller.go | 4 +- pkg/glance/cronjob.go | 13 ++++-- 3 files changed, 74 insertions(+), 13 deletions(-) diff --git a/controllers/glance_controller.go b/controllers/glance_controller.go index caf4744d2..300fbfa9b 100644 --- a/controllers/glance_controller.go +++ b/controllers/glance_controller.go @@ -51,6 +51,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/util" "github.com/openstack-k8s-operators/lib-common/modules/openstack" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -657,16 +658,10 @@ func (r *GlanceReconciler) reconcileNormal(ctx context.Context, instance *glance instance.Status.Conditions.Set(apiCondition) } - // NOTE: a CronJob is created at this stage to purge all soft deleted records. - // This command should be executed periodically to avoid glance database becomes - // bigger by getting filled by soft-deleted records. - cronjobDef := glance.CronJob(instance, serviceLabels, serviceAnnotations, glance.DBPurge) - cronjob := cronjob.NewCronJob( - cronjobDef, - 5*time.Second, - ) + // create CronJobs: DBPurge (always), CacheCleaner and CachePruner if image-cache + // is enabled - ctrlResult, err = cronjob.CreateOrPatch(ctx, helper) + ctrlResult, err = r.ensureCronJobs(ctx, helper, instance, serviceLabels, serviceAnnotations) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.CronJobReadyCondition, @@ -776,6 +771,20 @@ func (r *GlanceReconciler) generateServiceConfig( glance.DatabaseName, ), } + // We set in the main 00-config-default.conf the image-cache bits that will + // be used by CronJobs cleaner and pruner + if len(instance.Spec.ImageCacheSize) > 0 { + defer func() { + if err := recover(); err != nil { + r.Log.Error(err.(error), "ImageCacheSize not valid: %s") + } + }() + cacheSize := resource.MustParse(instance.Spec.ImageCacheSize) + templateParameters["CacheEnabled"] = true + templateParameters["CacheMaxSize"] = cacheSize.Value() + templateParameters["ImageCacheDir"] = glance.ImageCacheDir + } + customData := map[string]string{glance.CustomConfigFileName: instance.Spec.CustomServiceConfig} for key, data := range instance.Spec.DefaultConfigOverwrite { @@ -856,6 +865,49 @@ func (r *GlanceReconciler) ensurePVC( return ctrlResult, err } +// ensureCronJobs - Create the required CronJobs to clean DB entries and image-cache +// if enabled +func (r *GlanceReconciler) ensureCronJobs( + ctx context.Context, + h *helper.Helper, + instance *glancev1.Glance, + serviceLabels map[string]string, + serviceAnnotations map[string]string, +) (ctrl.Result, error) { + + // DBPurge cronjob is not optional and always created to purge all soft deleted records. + // This command should be executed periodically to avoid glance database becomes + // bigger by getting filled by soft-deleted records. + cronjobDef := glance.CronJob(instance, serviceLabels, serviceAnnotations, glance.DBPurge) + dbPurgeCronJob := cronjob.NewCronJob( + cronjobDef, + 5*time.Second, + ) + ctrlResult, err := dbPurgeCronJob.CreateOrPatch(ctx, h) + if err != nil { + return ctrlResult, err + } + + // If image-cache has been enabled, create two additional cronJobs: + // - CacheCleanerJob: clean stalled images or in an invalid state + // - CachePrunerJob: clean the image-cache folder to stay under ImageCacheSize + // limit + if len(instance.Spec.ImageCacheSize) > 0 { + for _, item := range []glance.CronJobType{glance.CacheCleaner, glance.CachePruner} { + cronjobDef = glance.CronJob(instance, serviceLabels, serviceAnnotations, item) + cronjob := cronjob.NewCronJob( + cronjobDef, + 5*time.Second, + ) + ctrlResult, err = cronjob.CreateOrPatch(ctx, h) + if err != nil { + return ctrlResult, err + } + } + } + return ctrlResult, err +} + // registeredLimitsDelete - cleanup registered limits in keystone func (r *GlanceReconciler) registeredLimitsDelete( ctx context.Context, diff --git a/controllers/glanceapi_controller.go b/controllers/glanceapi_controller.go index b76df76ba..b2d6efc82 100644 --- a/controllers/glanceapi_controller.go +++ b/controllers/glanceapi_controller.go @@ -23,6 +23,7 @@ import ( "time" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" @@ -753,8 +754,9 @@ func (r *GlanceAPIReconciler) generateServiceConfig( // Configure the cache bits accordingly as global options (00-config.conf) if len(instance.Spec.ImageCacheSize) > 0 { + cacheSize := resource.MustParse(instance.Spec.ImageCacheSize) templateParameters["CacheEnabled"] = true - templateParameters["CacheMaxSize"] = instance.Spec.ImageCacheSize + templateParameters["CacheMaxSize"] = cacheSize.Value() templateParameters["ImageCacheDir"] = glance.ImageCacheDir } diff --git a/pkg/glance/cronjob.go b/pkg/glance/cronjob.go index 989dc0da9..b2af1a708 100644 --- a/pkg/glance/cronjob.go +++ b/pkg/glance/cronjob.go @@ -18,6 +18,7 @@ package glance import ( glancev1 "github.com/openstack-k8s-operators/glance-operator/api/v1beta1" + "fmt" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -87,7 +88,6 @@ func CronJob( }, }, } - cronJobVolumeMounts := []corev1.VolumeMount{ { Name: "db-purge-config-data", @@ -95,10 +95,18 @@ func CronJob( ReadOnly: true, }, } + // If cache is provided, we expect the main glance_controller to request a + // PVC that should be used for that purpose (according to ImageCacheSize) + // and it should be available to the Cache CronJobs to properly clean the + // image-cache path + if cjType == "pruner" || cjType == "cleaner" { + cronJobVolume = append(cronJobVolume, GetCacheVolume(ServiceName+"-cache")...) + cronJobVolumeMounts = append(cronJobVolumeMounts, GetCacheVolumeMount()...) + } cronjob := &batchv1.CronJob{ ObjectMeta: metav1.ObjectMeta{ - Name: ServiceName + "-cron", + Name: fmt.Sprintf("%s-%s-cron", ServiceName, cjType), Namespace: instance.Namespace, }, Spec: batchv1.CronJobSpec{ @@ -140,6 +148,5 @@ func CronJob( if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 { cronjob.Spec.JobTemplate.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector } - return cronjob }