Enable image-cache for GlanceAPI

This patch is supposed to handle the image-cache feature for GlanceAPI
and it brings three main changes:

- A new PVC that will be bound to the image-cache dir is requested and
  created, and the apiVolume related arrays are updated to have this
  additional VolumeMount

- The GlanceAPISpec exposes an "imageCacheSize" parameter, which is
  passed to the top-level Glance CR and it's empty by default. If it's
  explicitly defined it means that a human operator is requesting cache
  for this component, enabling/triggering the related logic.

- The global config for glanceAPI has been updated to support all the
  cache related parameters in the DEFAULT section, hence new template
  parameters have been added as part of the cache enablement logic

It's still missing the logic of enabling cronJobs from the main
controller, but, given the size of the change, that will be part of a
follow up patch.

Signed-off-by: Francesco Pantano <[email protected]>

api/bases/glance.openstack.org_glanceapis.yaml

@@ -825,6 +825,9 @@ spec:
                   - extraVol
                   type: object
                 type: array
+              imageCacheSize:
+                default: ""
+                type: string
               networkAttachments:
                 items:
                   type: string
@@ -946,6 +949,7 @@ spec:
             required:
             - containerImage
             - databaseHostname
+            - imageCacheSize
             - secret
             - serviceAccount
             type: object

api/bases/glance.openstack.org_glances.yaml

@@ -1056,6 +1056,9 @@ spec:
                 required:
                 - containerImage
                 type: object
+              imageCacheSize:
+                default: ""
+                type: string
               nodeSelector:
                 additionalProperties:
                   type: string
@@ -1109,6 +1112,7 @@ spec:
             - databaseInstance
             - glanceAPIExternal
             - glanceAPIInternal
+            - imageCacheSize
             - secret
             - storageRequest
             type: object

api/v1beta1/glance_types.go

@@ -119,6 +119,10 @@ type GlanceSpec struct {
 	// Quotas is defined, per-tenant quotas are enforced according to the
 	// registered keystone limits
 	Quotas QuotaLimits `json:"quotas,omitempty"`
+
+	// ImageCacheSize, provides the size of the cache that will be reflected in the image_cache_max_size parameter
+	// +kubebuilder:default=""
+	ImageCacheSize string `json:"imageCacheSize"`
 }
 
 // PasswordSelector to identify the DB and AdminUser password from the Secret

api/v1beta1/glanceapi_types.go

@@ -77,6 +77,10 @@ type GlanceAPISpec struct {
 	// QuotaEnforce if true, per-tenant quotas are enforced according to the
 	// registered keystone limits
 	Quota bool `json:"quota"`
+
+	// ImageCacheSize, provides the size of the cache that will be reflected in the image_cache_max_size parameter
+	// +kubebuilder:default=""
+	ImageCacheSize string `json:"imageCacheSize"`
 }
 
 // GlanceAPIDebug defines the observed state of GlanceAPIDebug

config/crd/bases/glance.openstack.org_glanceapis.yaml

@@ -825,6 +825,9 @@ spec:
                   - extraVol
                   type: object
                 type: array
+              imageCacheSize:
+                default: ""
+                type: string
               networkAttachments:
                 items:
                   type: string
@@ -946,6 +949,7 @@ spec:
             required:
             - containerImage
             - databaseHostname
+            - imageCacheSize
             - secret
             - serviceAccount
             type: object

config/crd/bases/glance.openstack.org_glances.yaml

@@ -1056,6 +1056,9 @@ spec:
                 required:
                 - containerImage
                 type: object
+              imageCacheSize:
+                default: ""
+                type: string
               nodeSelector:
                 additionalProperties:
                   type: string
@@ -1109,6 +1112,7 @@ spec:
             - databaseInstance
             - glanceAPIExternal
             - glanceAPIInternal
+            - imageCacheSize
             - secret
             - storageRequest
             type: object

controllers/glance_controller.go

@@ -284,17 +284,8 @@ func (r *GlanceReconciler) reconcileInit(
 ) (ctrl.Result, error) {
 	r.Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name))
 
-	// Define a new PVC object
-	// TODO: Once conditions added to PVC lib-common logic, handle
-	//       the returned condition here
-	// TODO: This could be superfluous if backed by Ceph?
-	pvc := pvc.NewPvc(
-		glance.Pvc(instance, serviceLabels),
-		time.Duration(5)*time.Second,
-	)
-
-	ctrlResult, err := pvc.CreateOrPatch(ctx, helper)
-
+	// Define the PVCs objects required by Glance
+	ctrlResult, err := r.ensurePVC(ctx, helper, instance, serviceLabels)
 	if err != nil {
 		return ctrlResult, err
 	} else if (ctrlResult != ctrl.Result{}) {
@@ -707,6 +698,7 @@ func (r *GlanceReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, inst
 		ServiceUser:       instance.Spec.ServiceUser,
 		ServiceAccount:    instance.RbacResourceName(),
 		Quota:             instance.IsQuotaEnabled(),
+		ImageCacheSize:    instance.Spec.ImageCacheSize,
 	}
 
 	if apiSpec.GlanceAPITemplate.NodeSelector == nil {
@@ -832,6 +824,45 @@ func (r *GlanceReconciler) ensureRegisteredLimits(
 	return nil
 }
 
+// ensurePVC - Creates the PVCs required by Glance to start the GlanceAPI Pods
+func (r *GlanceReconciler) ensurePVC(
+	ctx context.Context,
+	h *helper.Helper,
+	instance *glancev1.Glance,
+	serviceLabels map[string]string,
+) (ctrl.Result, error) {
+	// Define a new PVC object
+	localPvc := pvc.NewPvc(
+		glance.Pvc(instance, serviceLabels, glance.PvcLocal),
+		time.Duration(5)*time.Second,
+	)
+
+	ctrlResult, err := localPvc.CreateOrPatch(ctx, h)
+
+	if err != nil {
+		return ctrlResult, err
+	} else if (ctrlResult != ctrl.Result{}) {
+		return ctrlResult, nil
+	}
+
+	// Handle an additional PVC creation an ImageCacheSize is provided
+	if len(instance.Spec.ImageCacheSize) > 0 {
+		cachePvc := pvc.NewPvc(
+			glance.Pvc(instance, serviceLabels, glance.PvcCache),
+			time.Duration(5)*time.Second,
+		)
+		ctrlResult, err := cachePvc.CreateOrPatch(ctx, h)
+
+		if err != nil {
+			return ctrlResult, err
+		} else if (ctrlResult != ctrl.Result{}) {
+			return ctrlResult, nil
+		}
+	}
+	// End PVC creation/patch
+	return ctrlResult, nil
+}
+
 // registeredLimitsDelete - cleanup registered limits in keystone
 func (r *GlanceReconciler) registeredLimitsDelete(
 	ctx context.Context,

controllers/glanceapi_controller.go

@@ -751,6 +751,13 @@ func (r *GlanceAPIReconciler) generateServiceConfig(
 		templateParameters["ShowMultipleLocations"] = false
 	}
 
+	// Configure the cache bits accordingly as global options (00-config.conf)
+	if len(instance.Spec.ImageCacheSize) > 0 {
+		templateParameters["CacheEnabled"] = true
+		templateParameters["CacheMaxSize"] = instance.Spec.ImageCacheSize
+		templateParameters["ImageCacheDir"] = glance.ImageCacheDir
+	}
+
 	// 00-default.conf will be regenerated as we have a ln -s of the
 	// templates/glance/config directory
 	// Do not generate -scripts as they are inherited from the top-level CR

pkg/glance/const.go

@@ -22,6 +22,9 @@ import (
 // CronJobType -
 type CronJobType string
 
+// PvcType -
+type PvcType string
+
 const (
 	// ServiceName -
 	ServiceName = "glance"
@@ -30,6 +33,11 @@ const (
 	// DatabaseName -
 	DatabaseName = "glance"
 
+	// PvcLocal for a generic glanceAPI instance
+	PvcLocal PvcType = "local"
+	// PvcCache is used to define a PVC mounted for image caching purposes
+	PvcCache PvcType = "cache"
+
 	// GlancePublicPort -
 	GlancePublicPort int32 = 9292
 	// GlanceInternalPort -
@@ -77,6 +85,8 @@ const (
 	CacheCleanerDefaultSchedule = "1 0 * * *"
 	//CachePrunerDefaultSchedule -
 	CachePrunerDefaultSchedule = "*/30 * * * *"
+	//ImageCacheDir -
+	ImageCacheDir = "/var/lib/glance/image-cache"
 )
 
 // DBPurgeCommandBase -

pkg/glance/pvc.go

@@ -1,17 +1,28 @@
 package glance
 
 import (
+	"fmt"
 	glancev1 "github.com/openstack-k8s-operators/glance-operator/api/v1beta1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // Pvc - creates and returns a PVC object for a backing store
-func Pvc(api *glancev1.Glance, labels map[string]string) *corev1.PersistentVolumeClaim {
+func Pvc(api *glancev1.Glance, labels map[string]string, pvcType PvcType) *corev1.PersistentVolumeClaim {
+	// By default we point to a local storage pvc request
+	// that will be customized in case the pvc is requested
+	// for cache purposes
+	requestSize := api.Spec.StorageRequest
+	pvcName := ServiceName
+	if pvcType == "cache" {
+		requestSize = api.Spec.ImageCacheSize
+		// append -cache to avoid confusion when listing PVCs
+		pvcName = fmt.Sprintf("%s-cache", ServiceName)
+	}
 	pvc := &corev1.PersistentVolumeClaim{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      ServiceName,
+			Name:      pvcName,
 			Namespace: api.Namespace,
 			Labels:    labels,
 		},
@@ -21,7 +32,7 @@ func Pvc(api *glancev1.Glance, labels map[string]string) *corev1.PersistentVolum
 			},
 			Resources: corev1.ResourceRequirements{
 				Requests: corev1.ResourceList{
-					corev1.ResourceStorage: resource.MustParse(api.Spec.StorageRequest),
+					corev1.ResourceStorage: resource.MustParse(requestSize),
 				},
 			},
 			StorageClassName: &api.Spec.StorageClass,

pkg/glance/volumes.go

@@ -280,3 +280,28 @@ func GetHttpdVolumeMount() []corev1.VolumeMount {
 		},
 	}
 }
+
+// GetCacheVolume - Return the Volume used for image caching purposes
+func GetCacheVolume(pvcName string) []corev1.Volume {
+	return []corev1.Volume{
+		{
+			Name: "image-cache",
+			VolumeSource: corev1.VolumeSource{
+				PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+					ClaimName: pvcName,
+				},
+			},
+		},
+	}
+}
+
+// GetCacheVolumeMount - Return the VolumeMount used for image caching purposes
+func GetCacheVolumeMount() []corev1.VolumeMount {
+	return []corev1.VolumeMount{
+		{
+			Name:      "image-cache",
+			MountPath: ImageCacheDir,
+			ReadOnly:  false,
+		},
+	}
+}

pkg/glanceapi/deployment.go

@@ -134,11 +134,17 @@ func Deployment(
 			ReadOnly:  true,
 		},
 	}
-
 	// Append LogVolume to the apiVolumes: this will be used to stream
 	// logging
 	apiVolumeMounts = append(apiVolumeMounts, glance.GetLogVolumeMount()...)
 
+	// If cache is provided, we expect the main glance_controller to request a
+	// PVC that should be used for that purpose (according to ImageCacheSize)
+	if len(instance.Spec.ImageCacheSize) > 0 {
+		apiVolumes = append(apiVolumes, glance.GetCacheVolume(glance.ServiceName+"-cache")...)
+		apiVolumeMounts = append(apiVolumeMounts, glance.GetCacheVolumeMount()...)
+	}
+
 	deployment := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf("%s-api", instance.Name),

templates/glance/config/00-config.conf

@@ -11,7 +11,6 @@ enabled_import_methods=[web-download]
 bind_host=127.0.0.1
 bind_port=9293
 workers=3
-image_cache_dir=/var/lib/glance/image-cache
 {{ if (index . "LogFile") }}
 # enable log rotation in oslo config by default
 max_logfile_count=5
@@ -20,6 +19,13 @@ log_rotation_type=size
 log_file = {{ .LogFile }}
 {{ end }}
 enabled_backends=default_backend:file
+# cache related parameters
+{{ if (index . "CacheEnabled") }}
+image_cache_dir = {{ .ImageCacheDir }}
+image_cache_max_size =  {{ .CacheMaxSize }}
+image_cache_stall_time = 86400
+{{ end }}
+
 {{ if (index . "QuotaEnabled") }}
 use_keystone_limits = {{ .QuotaEnabled }}
 
@@ -76,7 +82,7 @@ config_file = /etc/glance/glance-api-paste.ini
 flavor = keystone
 
 [os_glance_staging_store]
-filesystem_store_datadir = /var/lib/glance/os_glance_staging_store/
+filesystem_store_datadir = /var/lib/glance/os_glance_tasks_store/
 
 [os_glance_tasks_store]
 filesystem_store_datadir = /var/lib/glance/os_glance_tasks_store/