From fb33c84b70d7806369dc77790e02c6126d1f5417 Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Thu, 5 Oct 2023 13:20:08 +0200 Subject: [PATCH] Enable image-cache for GlanceAPI This patch is supposed to handle the image-cache feature for GlanceAPI and it brings three main changes: - A new PVC that will be bound to the image-cache dir is requested and created, and the apiVolume related arrays are updated to have this additional VolumeMount - The GlanceAPISpec exposes an "imageCacheSize" parameter, which is passed to the top-level Glance CR and it's empty by default. If it's explicitly defined it means that a human operator is requesting cache for this component, enabling/triggering the related logic. - The global config for glanceAPI has been updated to support all the cache related parameters in the DEFAULT section, hence new template parameters have been added as part of the cache enablement logic It's still missing the logic of enabling cronJobs from the main controller, but, given the size of the change, that will be part of a follow up patch. Signed-off-by: Francesco Pantano --- .../glance.openstack.org_glanceapis.yaml | 4 ++ api/bases/glance.openstack.org_glances.yaml | 4 ++ api/v1beta1/glance_types.go | 4 ++ api/v1beta1/glanceapi_types.go | 4 ++ .../glance.openstack.org_glanceapis.yaml | 4 ++ .../bases/glance.openstack.org_glances.yaml | 4 ++ controllers/glance_controller.go | 53 +++++++++++++++---- controllers/glanceapi_controller.go | 7 +++ pkg/glance/const.go | 10 ++++ pkg/glance/pvc.go | 17 ++++-- pkg/glance/volumes.go | 25 +++++++++ pkg/glanceapi/deployment.go | 8 ++- templates/glance/config/00-config.conf | 10 +++- 13 files changed, 137 insertions(+), 17 deletions(-) diff --git a/api/bases/glance.openstack.org_glanceapis.yaml b/api/bases/glance.openstack.org_glanceapis.yaml index 2bbed161e..5f6442323 100644 --- a/api/bases/glance.openstack.org_glanceapis.yaml +++ b/api/bases/glance.openstack.org_glanceapis.yaml @@ -825,6 +825,9 @@ spec: - extraVol type: object type: array + imageCacheSize: + default: "" + type: string networkAttachments: items: type: string @@ -946,6 +949,7 @@ spec: required: - containerImage - databaseHostname + - imageCacheSize - secret - serviceAccount type: object diff --git a/api/bases/glance.openstack.org_glances.yaml b/api/bases/glance.openstack.org_glances.yaml index d70e3ea6e..f89927018 100644 --- a/api/bases/glance.openstack.org_glances.yaml +++ b/api/bases/glance.openstack.org_glances.yaml @@ -1056,6 +1056,9 @@ spec: required: - containerImage type: object + imageCacheSize: + default: "" + type: string nodeSelector: additionalProperties: type: string @@ -1109,6 +1112,7 @@ spec: - databaseInstance - glanceAPIExternal - glanceAPIInternal + - imageCacheSize - secret - storageRequest type: object diff --git a/api/v1beta1/glance_types.go b/api/v1beta1/glance_types.go index 423e12877..050f76ad2 100644 --- a/api/v1beta1/glance_types.go +++ b/api/v1beta1/glance_types.go @@ -119,6 +119,10 @@ type GlanceSpec struct { // Quotas is defined, per-tenant quotas are enforced according to the // registered keystone limits Quotas QuotaLimits `json:"quotas,omitempty"` + + // ImageCacheSize, provides the size of the cache that will be reflected in the image_cache_max_size parameter + // +kubebuilder:default="" + ImageCacheSize string `json:"imageCacheSize"` } // PasswordSelector to identify the DB and AdminUser password from the Secret diff --git a/api/v1beta1/glanceapi_types.go b/api/v1beta1/glanceapi_types.go index d5d19f17a..238c2bce6 100644 --- a/api/v1beta1/glanceapi_types.go +++ b/api/v1beta1/glanceapi_types.go @@ -77,6 +77,10 @@ type GlanceAPISpec struct { // QuotaEnforce if true, per-tenant quotas are enforced according to the // registered keystone limits Quota bool `json:"quota"` + + // ImageCacheSize, provides the size of the cache that will be reflected in the image_cache_max_size parameter + // +kubebuilder:default="" + ImageCacheSize string `json:"imageCacheSize"` } // GlanceAPIDebug defines the observed state of GlanceAPIDebug diff --git a/config/crd/bases/glance.openstack.org_glanceapis.yaml b/config/crd/bases/glance.openstack.org_glanceapis.yaml index 2bbed161e..5f6442323 100644 --- a/config/crd/bases/glance.openstack.org_glanceapis.yaml +++ b/config/crd/bases/glance.openstack.org_glanceapis.yaml @@ -825,6 +825,9 @@ spec: - extraVol type: object type: array + imageCacheSize: + default: "" + type: string networkAttachments: items: type: string @@ -946,6 +949,7 @@ spec: required: - containerImage - databaseHostname + - imageCacheSize - secret - serviceAccount type: object diff --git a/config/crd/bases/glance.openstack.org_glances.yaml b/config/crd/bases/glance.openstack.org_glances.yaml index d70e3ea6e..f89927018 100644 --- a/config/crd/bases/glance.openstack.org_glances.yaml +++ b/config/crd/bases/glance.openstack.org_glances.yaml @@ -1056,6 +1056,9 @@ spec: required: - containerImage type: object + imageCacheSize: + default: "" + type: string nodeSelector: additionalProperties: type: string @@ -1109,6 +1112,7 @@ spec: - databaseInstance - glanceAPIExternal - glanceAPIInternal + - imageCacheSize - secret - storageRequest type: object diff --git a/controllers/glance_controller.go b/controllers/glance_controller.go index 4e370f6ed..127742d85 100644 --- a/controllers/glance_controller.go +++ b/controllers/glance_controller.go @@ -284,17 +284,8 @@ func (r *GlanceReconciler) reconcileInit( ) (ctrl.Result, error) { r.Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name)) - // Define a new PVC object - // TODO: Once conditions added to PVC lib-common logic, handle - // the returned condition here - // TODO: This could be superfluous if backed by Ceph? - pvc := pvc.NewPvc( - glance.Pvc(instance, serviceLabels), - time.Duration(5)*time.Second, - ) - - ctrlResult, err := pvc.CreateOrPatch(ctx, helper) - + // Define the PVCs objects required by Glance + ctrlResult, err := r.ensurePVC(ctx, helper, instance, serviceLabels) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -707,6 +698,7 @@ func (r *GlanceReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, inst ServiceUser: instance.Spec.ServiceUser, ServiceAccount: instance.RbacResourceName(), Quota: instance.IsQuotaEnabled(), + ImageCacheSize: instance.Spec.ImageCacheSize, } if apiSpec.GlanceAPITemplate.NodeSelector == nil { @@ -832,6 +824,45 @@ func (r *GlanceReconciler) ensureRegisteredLimits( return nil } +// ensurePVC - Creates the PVCs required by Glance to start the GlanceAPI Pods +func (r *GlanceReconciler) ensurePVC( + ctx context.Context, + h *helper.Helper, + instance *glancev1.Glance, + serviceLabels map[string]string, +) (ctrl.Result, error) { + // Define a new PVC object + localPvc := pvc.NewPvc( + glance.Pvc(instance, serviceLabels, glance.PvcLocal), + time.Duration(5)*time.Second, + ) + + ctrlResult, err := localPvc.CreateOrPatch(ctx, h) + + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // Handle an additional PVC creation an ImageCacheSize is provided + if len(instance.Spec.ImageCacheSize) > 0 { + cachePvc := pvc.NewPvc( + glance.Pvc(instance, serviceLabels, glance.PvcCache), + time.Duration(5)*time.Second, + ) + ctrlResult, err := cachePvc.CreateOrPatch(ctx, h) + + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + } + // End PVC creation/patch + return ctrlResult, nil +} + // registeredLimitsDelete - cleanup registered limits in keystone func (r *GlanceReconciler) registeredLimitsDelete( ctx context.Context, diff --git a/controllers/glanceapi_controller.go b/controllers/glanceapi_controller.go index 84d5d803f..b76df76ba 100644 --- a/controllers/glanceapi_controller.go +++ b/controllers/glanceapi_controller.go @@ -751,6 +751,13 @@ func (r *GlanceAPIReconciler) generateServiceConfig( templateParameters["ShowMultipleLocations"] = false } + // Configure the cache bits accordingly as global options (00-config.conf) + if len(instance.Spec.ImageCacheSize) > 0 { + templateParameters["CacheEnabled"] = true + templateParameters["CacheMaxSize"] = instance.Spec.ImageCacheSize + templateParameters["ImageCacheDir"] = glance.ImageCacheDir + } + // 00-default.conf will be regenerated as we have a ln -s of the // templates/glance/config directory // Do not generate -scripts as they are inherited from the top-level CR diff --git a/pkg/glance/const.go b/pkg/glance/const.go index 803eda45c..af1127e79 100644 --- a/pkg/glance/const.go +++ b/pkg/glance/const.go @@ -22,6 +22,9 @@ import ( // CronJobType - type CronJobType string +// PvcType - +type PvcType string + const ( // ServiceName - ServiceName = "glance" @@ -30,6 +33,11 @@ const ( // DatabaseName - DatabaseName = "glance" + // PvcLocal for a generic glanceAPI instance + PvcLocal PvcType = "local" + // PvcCache is used to define a PVC mounted for image caching purposes + PvcCache PvcType = "cache" + // GlancePublicPort - GlancePublicPort int32 = 9292 // GlanceInternalPort - @@ -77,6 +85,8 @@ const ( CacheCleanerDefaultSchedule = "1 0 * * *" //CachePrunerDefaultSchedule - CachePrunerDefaultSchedule = "*/30 * * * *" + //ImageCacheDir - + ImageCacheDir = "/var/lib/glance/image-cache" ) // DBPurgeCommandBase - diff --git a/pkg/glance/pvc.go b/pkg/glance/pvc.go index 3ec2fa0dd..bdacb187f 100644 --- a/pkg/glance/pvc.go +++ b/pkg/glance/pvc.go @@ -1,6 +1,7 @@ package glance import ( + "fmt" glancev1 "github.com/openstack-k8s-operators/glance-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -8,10 +9,20 @@ import ( ) // Pvc - creates and returns a PVC object for a backing store -func Pvc(api *glancev1.Glance, labels map[string]string) *corev1.PersistentVolumeClaim { +func Pvc(api *glancev1.Glance, labels map[string]string, pvcType PvcType) *corev1.PersistentVolumeClaim { + // By default we point to a local storage pvc request + // that will be customized in case the pvc is requested + // for cache purposes + requestSize := api.Spec.StorageRequest + pvcName := ServiceName + if pvcType == "cache" { + requestSize = api.Spec.ImageCacheSize + // append -cache to avoid confusion when listing PVCs + pvcName = fmt.Sprintf("%s-cache", ServiceName) + } pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: ServiceName, + Name: pvcName, Namespace: api.Namespace, Labels: labels, }, @@ -21,7 +32,7 @@ func Pvc(api *glancev1.Glance, labels map[string]string) *corev1.PersistentVolum }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse(api.Spec.StorageRequest), + corev1.ResourceStorage: resource.MustParse(requestSize), }, }, StorageClassName: &api.Spec.StorageClass, diff --git a/pkg/glance/volumes.go b/pkg/glance/volumes.go index 47aefd3c1..0e03e15da 100644 --- a/pkg/glance/volumes.go +++ b/pkg/glance/volumes.go @@ -280,3 +280,28 @@ func GetHttpdVolumeMount() []corev1.VolumeMount { }, } } + +// GetCacheVolume - Return the Volume used for image caching purposes +func GetCacheVolume(pvcName string) []corev1.Volume { + return []corev1.Volume{ + { + Name: "image-cache", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + }, + }, + }, + } +} + +// GetCacheVolumeMount - Return the VolumeMount used for image caching purposes +func GetCacheVolumeMount() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: "image-cache", + MountPath: ImageCacheDir, + ReadOnly: false, + }, + } +} diff --git a/pkg/glanceapi/deployment.go b/pkg/glanceapi/deployment.go index cab52d391..b3dbb2f7c 100644 --- a/pkg/glanceapi/deployment.go +++ b/pkg/glanceapi/deployment.go @@ -134,11 +134,17 @@ func Deployment( ReadOnly: true, }, } - // Append LogVolume to the apiVolumes: this will be used to stream // logging apiVolumeMounts = append(apiVolumeMounts, glance.GetLogVolumeMount()...) + // If cache is provided, we expect the main glance_controller to request a + // PVC that should be used for that purpose (according to ImageCacheSize) + if len(instance.Spec.ImageCacheSize) > 0 { + apiVolumes = append(apiVolumes, glance.GetCacheVolume(glance.ServiceName+"-cache")...) + apiVolumeMounts = append(apiVolumeMounts, glance.GetCacheVolumeMount()...) + } + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-api", instance.Name), diff --git a/templates/glance/config/00-config.conf b/templates/glance/config/00-config.conf index 0ae666ca0..735bf2605 100644 --- a/templates/glance/config/00-config.conf +++ b/templates/glance/config/00-config.conf @@ -11,7 +11,6 @@ enabled_import_methods=[web-download] bind_host=127.0.0.1 bind_port=9293 workers=3 -image_cache_dir=/var/lib/glance/image-cache {{ if (index . "LogFile") }} # enable log rotation in oslo config by default max_logfile_count=5 @@ -20,6 +19,13 @@ log_rotation_type=size log_file = {{ .LogFile }} {{ end }} enabled_backends=default_backend:file +# cache related parameters +{{ if (index . "CacheEnabled") }} +image_cache_dir = {{ .ImageCacheDir }} +image_cache_max_size = {{ .CacheMaxSize }} +image_cache_stall_time = 86400 +{{ end }} + {{ if (index . "QuotaEnabled") }} use_keystone_limits = {{ .QuotaEnabled }} @@ -76,7 +82,7 @@ config_file = /etc/glance/glance-api-paste.ini flavor = keystone [os_glance_staging_store] -filesystem_store_datadir = /var/lib/glance/os_glance_staging_store/ +filesystem_store_datadir = /var/lib/glance/os_glance_tasks_store/ [os_glance_tasks_store] filesystem_store_datadir = /var/lib/glance/os_glance_tasks_store/