diff --git a/api/bases/glance.openstack.org_glanceapis.yaml b/api/bases/glance.openstack.org_glanceapis.yaml index 2bbed161e..5f6442323 100644 --- a/api/bases/glance.openstack.org_glanceapis.yaml +++ b/api/bases/glance.openstack.org_glanceapis.yaml @@ -825,6 +825,9 @@ spec: - extraVol type: object type: array + imageCacheSize: + default: "" + type: string networkAttachments: items: type: string @@ -946,6 +949,7 @@ spec: required: - containerImage - databaseHostname + - imageCacheSize - secret - serviceAccount type: object diff --git a/api/bases/glance.openstack.org_glances.yaml b/api/bases/glance.openstack.org_glances.yaml index d70e3ea6e..f89927018 100644 --- a/api/bases/glance.openstack.org_glances.yaml +++ b/api/bases/glance.openstack.org_glances.yaml @@ -1056,6 +1056,9 @@ spec: required: - containerImage type: object + imageCacheSize: + default: "" + type: string nodeSelector: additionalProperties: type: string @@ -1109,6 +1112,7 @@ spec: - databaseInstance - glanceAPIExternal - glanceAPIInternal + - imageCacheSize - secret - storageRequest type: object diff --git a/api/v1beta1/glance_types.go b/api/v1beta1/glance_types.go index 423e12877..050f76ad2 100644 --- a/api/v1beta1/glance_types.go +++ b/api/v1beta1/glance_types.go @@ -119,6 +119,10 @@ type GlanceSpec struct { // Quotas is defined, per-tenant quotas are enforced according to the // registered keystone limits Quotas QuotaLimits `json:"quotas,omitempty"` + + // ImageCacheSize, provides the size of the cache that will be reflected in the image_cache_max_size parameter + // +kubebuilder:default="" + ImageCacheSize string `json:"imageCacheSize"` } // PasswordSelector to identify the DB and AdminUser password from the Secret diff --git a/api/v1beta1/glanceapi_types.go b/api/v1beta1/glanceapi_types.go index d5d19f17a..238c2bce6 100644 --- a/api/v1beta1/glanceapi_types.go +++ b/api/v1beta1/glanceapi_types.go @@ -77,6 +77,10 @@ type GlanceAPISpec struct { // QuotaEnforce if true, per-tenant quotas are enforced according to the // registered keystone limits Quota bool `json:"quota"` + + // ImageCacheSize, provides the size of the cache that will be reflected in the image_cache_max_size parameter + // +kubebuilder:default="" + ImageCacheSize string `json:"imageCacheSize"` } // GlanceAPIDebug defines the observed state of GlanceAPIDebug diff --git a/config/crd/bases/glance.openstack.org_glanceapis.yaml b/config/crd/bases/glance.openstack.org_glanceapis.yaml index 2bbed161e..5f6442323 100644 --- a/config/crd/bases/glance.openstack.org_glanceapis.yaml +++ b/config/crd/bases/glance.openstack.org_glanceapis.yaml @@ -825,6 +825,9 @@ spec: - extraVol type: object type: array + imageCacheSize: + default: "" + type: string networkAttachments: items: type: string @@ -946,6 +949,7 @@ spec: required: - containerImage - databaseHostname + - imageCacheSize - secret - serviceAccount type: object diff --git a/config/crd/bases/glance.openstack.org_glances.yaml b/config/crd/bases/glance.openstack.org_glances.yaml index d70e3ea6e..f89927018 100644 --- a/config/crd/bases/glance.openstack.org_glances.yaml +++ b/config/crd/bases/glance.openstack.org_glances.yaml @@ -1056,6 +1056,9 @@ spec: required: - containerImage type: object + imageCacheSize: + default: "" + type: string nodeSelector: additionalProperties: type: string @@ -1109,6 +1112,7 @@ spec: - databaseInstance - glanceAPIExternal - glanceAPIInternal + - imageCacheSize - secret - storageRequest type: object diff --git a/controllers/glance_controller.go b/controllers/glance_controller.go index 4e370f6ed..127742d85 100644 --- a/controllers/glance_controller.go +++ b/controllers/glance_controller.go @@ -284,17 +284,8 @@ func (r *GlanceReconciler) reconcileInit( ) (ctrl.Result, error) { r.Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name)) - // Define a new PVC object - // TODO: Once conditions added to PVC lib-common logic, handle - // the returned condition here - // TODO: This could be superfluous if backed by Ceph? - pvc := pvc.NewPvc( - glance.Pvc(instance, serviceLabels), - time.Duration(5)*time.Second, - ) - - ctrlResult, err := pvc.CreateOrPatch(ctx, helper) - + // Define the PVCs objects required by Glance + ctrlResult, err := r.ensurePVC(ctx, helper, instance, serviceLabels) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -707,6 +698,7 @@ func (r *GlanceReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, inst ServiceUser: instance.Spec.ServiceUser, ServiceAccount: instance.RbacResourceName(), Quota: instance.IsQuotaEnabled(), + ImageCacheSize: instance.Spec.ImageCacheSize, } if apiSpec.GlanceAPITemplate.NodeSelector == nil { @@ -832,6 +824,45 @@ func (r *GlanceReconciler) ensureRegisteredLimits( return nil } +// ensurePVC - Creates the PVCs required by Glance to start the GlanceAPI Pods +func (r *GlanceReconciler) ensurePVC( + ctx context.Context, + h *helper.Helper, + instance *glancev1.Glance, + serviceLabels map[string]string, +) (ctrl.Result, error) { + // Define a new PVC object + localPvc := pvc.NewPvc( + glance.Pvc(instance, serviceLabels, glance.PvcLocal), + time.Duration(5)*time.Second, + ) + + ctrlResult, err := localPvc.CreateOrPatch(ctx, h) + + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // Handle an additional PVC creation an ImageCacheSize is provided + if len(instance.Spec.ImageCacheSize) > 0 { + cachePvc := pvc.NewPvc( + glance.Pvc(instance, serviceLabels, glance.PvcCache), + time.Duration(5)*time.Second, + ) + ctrlResult, err := cachePvc.CreateOrPatch(ctx, h) + + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + } + // End PVC creation/patch + return ctrlResult, nil +} + // registeredLimitsDelete - cleanup registered limits in keystone func (r *GlanceReconciler) registeredLimitsDelete( ctx context.Context, diff --git a/controllers/glanceapi_controller.go b/controllers/glanceapi_controller.go index 84d5d803f..b76df76ba 100644 --- a/controllers/glanceapi_controller.go +++ b/controllers/glanceapi_controller.go @@ -751,6 +751,13 @@ func (r *GlanceAPIReconciler) generateServiceConfig( templateParameters["ShowMultipleLocations"] = false } + // Configure the cache bits accordingly as global options (00-config.conf) + if len(instance.Spec.ImageCacheSize) > 0 { + templateParameters["CacheEnabled"] = true + templateParameters["CacheMaxSize"] = instance.Spec.ImageCacheSize + templateParameters["ImageCacheDir"] = glance.ImageCacheDir + } + // 00-default.conf will be regenerated as we have a ln -s of the // templates/glance/config directory // Do not generate -scripts as they are inherited from the top-level CR diff --git a/pkg/glance/const.go b/pkg/glance/const.go index 803eda45c..af1127e79 100644 --- a/pkg/glance/const.go +++ b/pkg/glance/const.go @@ -22,6 +22,9 @@ import ( // CronJobType - type CronJobType string +// PvcType - +type PvcType string + const ( // ServiceName - ServiceName = "glance" @@ -30,6 +33,11 @@ const ( // DatabaseName - DatabaseName = "glance" + // PvcLocal for a generic glanceAPI instance + PvcLocal PvcType = "local" + // PvcCache is used to define a PVC mounted for image caching purposes + PvcCache PvcType = "cache" + // GlancePublicPort - GlancePublicPort int32 = 9292 // GlanceInternalPort - @@ -77,6 +85,8 @@ const ( CacheCleanerDefaultSchedule = "1 0 * * *" //CachePrunerDefaultSchedule - CachePrunerDefaultSchedule = "*/30 * * * *" + //ImageCacheDir - + ImageCacheDir = "/var/lib/glance/image-cache" ) // DBPurgeCommandBase - diff --git a/pkg/glance/pvc.go b/pkg/glance/pvc.go index 3ec2fa0dd..bdacb187f 100644 --- a/pkg/glance/pvc.go +++ b/pkg/glance/pvc.go @@ -1,6 +1,7 @@ package glance import ( + "fmt" glancev1 "github.com/openstack-k8s-operators/glance-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -8,10 +9,20 @@ import ( ) // Pvc - creates and returns a PVC object for a backing store -func Pvc(api *glancev1.Glance, labels map[string]string) *corev1.PersistentVolumeClaim { +func Pvc(api *glancev1.Glance, labels map[string]string, pvcType PvcType) *corev1.PersistentVolumeClaim { + // By default we point to a local storage pvc request + // that will be customized in case the pvc is requested + // for cache purposes + requestSize := api.Spec.StorageRequest + pvcName := ServiceName + if pvcType == "cache" { + requestSize = api.Spec.ImageCacheSize + // append -cache to avoid confusion when listing PVCs + pvcName = fmt.Sprintf("%s-cache", ServiceName) + } pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: ServiceName, + Name: pvcName, Namespace: api.Namespace, Labels: labels, }, @@ -21,7 +32,7 @@ func Pvc(api *glancev1.Glance, labels map[string]string) *corev1.PersistentVolum }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse(api.Spec.StorageRequest), + corev1.ResourceStorage: resource.MustParse(requestSize), }, }, StorageClassName: &api.Spec.StorageClass, diff --git a/pkg/glance/volumes.go b/pkg/glance/volumes.go index 47aefd3c1..0e03e15da 100644 --- a/pkg/glance/volumes.go +++ b/pkg/glance/volumes.go @@ -280,3 +280,28 @@ func GetHttpdVolumeMount() []corev1.VolumeMount { }, } } + +// GetCacheVolume - Return the Volume used for image caching purposes +func GetCacheVolume(pvcName string) []corev1.Volume { + return []corev1.Volume{ + { + Name: "image-cache", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + }, + }, + }, + } +} + +// GetCacheVolumeMount - Return the VolumeMount used for image caching purposes +func GetCacheVolumeMount() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: "image-cache", + MountPath: ImageCacheDir, + ReadOnly: false, + }, + } +} diff --git a/pkg/glanceapi/deployment.go b/pkg/glanceapi/deployment.go index cab52d391..b3dbb2f7c 100644 --- a/pkg/glanceapi/deployment.go +++ b/pkg/glanceapi/deployment.go @@ -134,11 +134,17 @@ func Deployment( ReadOnly: true, }, } - // Append LogVolume to the apiVolumes: this will be used to stream // logging apiVolumeMounts = append(apiVolumeMounts, glance.GetLogVolumeMount()...) + // If cache is provided, we expect the main glance_controller to request a + // PVC that should be used for that purpose (according to ImageCacheSize) + if len(instance.Spec.ImageCacheSize) > 0 { + apiVolumes = append(apiVolumes, glance.GetCacheVolume(glance.ServiceName+"-cache")...) + apiVolumeMounts = append(apiVolumeMounts, glance.GetCacheVolumeMount()...) + } + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-api", instance.Name), diff --git a/templates/glance/config/00-config.conf b/templates/glance/config/00-config.conf index 0ae666ca0..735bf2605 100644 --- a/templates/glance/config/00-config.conf +++ b/templates/glance/config/00-config.conf @@ -11,7 +11,6 @@ enabled_import_methods=[web-download] bind_host=127.0.0.1 bind_port=9293 workers=3 -image_cache_dir=/var/lib/glance/image-cache {{ if (index . "LogFile") }} # enable log rotation in oslo config by default max_logfile_count=5 @@ -20,6 +19,13 @@ log_rotation_type=size log_file = {{ .LogFile }} {{ end }} enabled_backends=default_backend:file +# cache related parameters +{{ if (index . "CacheEnabled") }} +image_cache_dir = {{ .ImageCacheDir }} +image_cache_max_size = {{ .CacheMaxSize }} +image_cache_stall_time = 86400 +{{ end }} + {{ if (index . "QuotaEnabled") }} use_keystone_limits = {{ .QuotaEnabled }} @@ -76,7 +82,7 @@ config_file = /etc/glance/glance-api-paste.ini flavor = keystone [os_glance_staging_store] -filesystem_store_datadir = /var/lib/glance/os_glance_staging_store/ +filesystem_store_datadir = /var/lib/glance/os_glance_tasks_store/ [os_glance_tasks_store] filesystem_store_datadir = /var/lib/glance/os_glance_tasks_store/