From d28ca0a79836ac885b375c94dd5e0bfece7629c3 Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Mon, 9 Sep 2024 22:29:52 +0200 Subject: [PATCH] Run httpd using kolla Instead of running the httpd -DFOREGROUND command as entrypoint for the -httpd sidecar container, this change moves the file copy and deployment logic to kolla. This is a requirement to not run the container as root user, because kolla helps to apply the right permissions to the config files (and pid) used by the process. The switch from root user to GlanceUID (already present as const) will be part of a different patch. Signed-off-by: Francesco Pantano --- pkg/glance/volumes.go | 55 +++++++++++++++---- pkg/glanceapi/statefulset.go | 49 ++--------------- .../glanceapi/config/glance-api-config.json | 16 ------ .../glanceapi/config/glance-httpd-config.json | 49 +++++++++++++++++ templates/glanceapi/config/httpd.conf | 1 + test/functional/glanceapi_controller_test.go | 2 +- test/kuttl/tests/glance_single/01-assert.yaml | 4 +- .../tests/glance_single_tls/01-assert.yaml | 23 +++----- test/kuttl/tests/glance_split/01-assert.yaml | 8 +-- 9 files changed, 117 insertions(+), 90 deletions(-) create mode 100644 templates/glanceapi/config/glance-httpd-config.json diff --git a/pkg/glance/volumes.go b/pkg/glance/volumes.go index 8953a198..16f6bce4 100644 --- a/pkg/glance/volumes.go +++ b/pkg/glance/volumes.go @@ -268,20 +268,13 @@ func GetHttpdVolumeMount() []corev1.VolumeMount { return []corev1.VolumeMount{ { Name: "config-data", - MountPath: "/etc/httpd/conf/httpd.conf", - SubPath: "httpd.conf", - ReadOnly: true, - }, - { - Name: "config-data", - MountPath: "/etc/httpd/conf.d/10-glance.conf", - SubPath: "10-glance-httpd.conf", + MountPath: "/var/lib/config-data/default", ReadOnly: true, }, { Name: "config-data", - MountPath: "/etc/httpd/conf.d/ssl.conf", - SubPath: "ssl.conf", + MountPath: "/var/lib/kolla/config_files/config.json", + SubPath: "glance-httpd-config.json", ReadOnly: true, }, } @@ -339,3 +332,45 @@ func GetScriptVolumeMount() []corev1.VolumeMount { }, } } + +// GetAPIVolumes - +func GetAPIVolumes(name string) []corev1.Volume { + var config0644AccessMode int32 = 0644 + apiVolumes := []corev1.Volume{ + { + Name: "config-data-custom", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", + }, + }, + }, + } + // Append LogVolume to the apiVolumes: this will be used to stream logging + apiVolumes = append(apiVolumes, GetLogVolume()...) + apiVolumes = append(apiVolumes, GetScriptVolume()...) + return apiVolumes +} + +// GetAPIVolumeMount - +func GetAPIVolumeMount(cacheSize string) []corev1.VolumeMount { + apiVolumeMounts := []corev1.VolumeMount{ + { + Name: "config-data", + MountPath: "/var/lib/kolla/config_files/config.json", + SubPath: "glance-api-config.json", + ReadOnly: true, + }, + } + // Append LogVolume to apiVolumes: this will be used to stream logging + apiVolumeMounts = append(apiVolumeMounts, GetLogVolumeMount()...) + // Append ScriptsVolume to apiVolumes + apiVolumeMounts = append(apiVolumeMounts, GetScriptVolumeMount()...) + // If cache is provided, we expect the main glance_controller to request a + // PVC that should be used for that purpose (according to ImageCache.Size) + if len(cacheSize) > 0 { + apiVolumeMounts = append(apiVolumeMounts, GetCacheVolumeMount()...) + } + return apiVolumeMounts +} diff --git a/pkg/glanceapi/statefulset.go b/pkg/glanceapi/statefulset.go index 18908116..0d73dde4 100644 --- a/pkg/glanceapi/statefulset.go +++ b/pkg/glanceapi/statefulset.go @@ -38,10 +38,8 @@ import ( ) const ( - // GlanceAPIServiceCommand - - GlanceAPIServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" - // GlanceAPIHttpdCommand - - GlanceAPIHttpdCommand = "/usr/sbin/httpd -DFOREGROUND" + // GlanceServiceCommand - + GlanceServiceCommand = "/usr/local/bin/kolla_start" ) // StatefulSet func @@ -54,8 +52,6 @@ func StatefulSet( ) (*appsv1.StatefulSet, error) { runAsUser := int64(0) - var config0644AccessMode int32 = 0644 - startupProbe := &corev1.Probe{ FailureThreshold: 6, PeriodSeconds: 10, @@ -111,41 +107,8 @@ func StatefulSet( envVars["GLANCE_DOMAIN"] = env.SetValue(instance.Status.Domain) envVars["URISCHEME"] = env.SetValue(string(glanceURIScheme)) - apiVolumes := []corev1.Volume{ - { - Name: "config-data-custom", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - DefaultMode: &config0644AccessMode, - SecretName: instance.Name + "-config-data", - }, - }, - }, - } - // Append LogVolume to the apiVolumes: this will be used to stream - // logging - apiVolumes = append(apiVolumes, glance.GetLogVolume()...) - apiVolumeMounts := []corev1.VolumeMount{ - { - Name: "config-data", - MountPath: "/var/lib/kolla/config_files/config.json", - SubPath: "glance-api-config.json", - ReadOnly: true, - }, - } - - // Append LogVolume to the apiVolumes: this will be used to stream logging - apiVolumeMounts = append(apiVolumeMounts, glance.GetLogVolumeMount()...) - - // Append scripts - apiVolumes = append(apiVolumes, glance.GetScriptVolume()...) - apiVolumeMounts = append(apiVolumeMounts, glance.GetScriptVolumeMount()...) - - // If cache is provided, we expect the main glance_controller to request a - // PVC that should be used for that purpose (according to ImageCacheSize) - if len(instance.Spec.ImageCache.Size) > 0 { - apiVolumeMounts = append(apiVolumeMounts, glance.GetCacheVolumeMount()...) - } + apiVolumes := glance.GetAPIVolumes(instance.Name) + apiVolumeMounts := glance.GetAPIVolumeMount(instance.Spec.ImageCache.Size) extraVolPropagation := append(glance.GlanceAPIPropagation, storage.PropagationType(instance.APIName())) @@ -255,7 +218,7 @@ func StatefulSet( "--", "/bin/bash", "-c", - string(GlanceAPIHttpdCommand), + string(GlanceServiceCommand), }, Image: instance.Spec.ContainerImage, SecurityContext: &corev1.SecurityContext{ @@ -278,7 +241,7 @@ func StatefulSet( "--", "/bin/bash", "-c", - string(GlanceAPIServiceCommand), + string(GlanceServiceCommand), }, Image: instance.Spec.ContainerImage, SecurityContext: &corev1.SecurityContext{ diff --git a/templates/glanceapi/config/glance-api-config.json b/templates/glanceapi/config/glance-api-config.json index 9b6bd608..996d8b24 100644 --- a/templates/glanceapi/config/glance-api-config.json +++ b/templates/glanceapi/config/glance-api-config.json @@ -56,22 +56,6 @@ "owner": "root:root", "perm": "0755" }, - { - "source": "/var/lib/config-data/tls/certs/*", - "dest": "/etc/pki/tls/certs/", - "owner": "root", - "perm": "0640", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/tls/private/*", - "dest": "/etc/pki/tls/private/", - "owner": "root", - "perm": "0600", - "optional": true, - "merge": true - }, { "source": "/usr/local/bin/container-scripts/kolla_extend_start", "dest": "/usr/local/bin/kolla_extend_start", diff --git a/templates/glanceapi/config/glance-httpd-config.json b/templates/glanceapi/config/glance-httpd-config.json new file mode 100644 index 00000000..109b5b52 --- /dev/null +++ b/templates/glanceapi/config/glance-httpd-config.json @@ -0,0 +1,49 @@ +{ + "command": "/usr/sbin/httpd -DFOREGROUND", + "config_files": [ + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "glance:glance", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "glance:glance", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/default/httpd.conf", + "dest": "/etc/httpd/conf/httpd.conf", + "owner": "glance:apache", + "optional": true, + "perm": "0644" + }, + { + "source": "/var/lib/config-data/default/10-glance-httpd.conf", + "dest": "/etc/httpd/conf.d/10-glance.conf", + "owner": "glance:apache", + "optional": true, + "perm": "0644" + }, + { + "source": "/var/lib/config-data/default/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "glance:apache", + "optional": true, + "perm": "0644" + } + ], + "permissions": [ + { + "path": "/etc/httpd/run", + "owner": "glance:apache", + "recurse": true + } + ] +} diff --git a/templates/glanceapi/config/httpd.conf b/templates/glanceapi/config/httpd.conf index 75c972c0..cd06bf92 100644 --- a/templates/glanceapi/config/httpd.conf +++ b/templates/glanceapi/config/httpd.conf @@ -19,5 +19,6 @@ LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-A SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded CustomLog /dev/stdout combined env=!forwarded CustomLog /dev/stdout proxy env=forwarded +ErrorLog /dev/stdout Include conf.d/10-glance.conf diff --git a/test/functional/glanceapi_controller_test.go b/test/functional/glanceapi_controller_test.go index 01c397e0..d504f7a3 100644 --- a/test/functional/glanceapi_controller_test.go +++ b/test/functional/glanceapi_controller_test.go @@ -217,7 +217,7 @@ var _ = Describe("Glanceapi controller", func() { // Check the glance-httpd container container = ss.Spec.Template.Spec.Containers[1] - Expect(container.VolumeMounts).To(HaveLen(3)) + Expect(container.VolumeMounts).To(HaveLen(2)) Expect(container.Image).To(Equal(glanceTest.ContainerImage)) // Check the glance-log container diff --git a/test/kuttl/tests/glance_single/01-assert.yaml b/test/kuttl/tests/glance_single/01-assert.yaml index d0a9cf98..1be4a233 100644 --- a/test/kuttl/tests/glance_single/01-assert.yaml +++ b/test/kuttl/tests/glance_single/01-assert.yaml @@ -68,7 +68,7 @@ spec: - -- - /bin/bash - -c - - /usr/sbin/httpd -DFOREGROUND + - /usr/local/bin/kolla_start command: - /usr/bin/dumb-init name: glance-httpd @@ -77,7 +77,7 @@ spec: - -- - /bin/bash - -c - - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start + - /usr/local/bin/kolla_start command: - /usr/bin/dumb-init name: glance-api diff --git a/test/kuttl/tests/glance_single_tls/01-assert.yaml b/test/kuttl/tests/glance_single_tls/01-assert.yaml index ab021b29..add52b56 100644 --- a/test/kuttl/tests/glance_single_tls/01-assert.yaml +++ b/test/kuttl/tests/glance_single_tls/01-assert.yaml @@ -66,20 +66,8 @@ spec: - -- - /bin/bash - -c - - /usr/sbin/httpd -DFOREGROUND + - /usr/local/bin/kolla_start volumeMounts: - - mountPath: /etc/httpd/conf/httpd.conf - name: config-data - readOnly: true - subPath: httpd.conf - - mountPath: /etc/httpd/conf.d/10-glance.conf - name: config-data - readOnly: true - subPath: 10-glance-httpd.conf - - mountPath: /etc/httpd/conf.d/ssl.conf - name: config-data - readOnly: true - subPath: ssl.conf - mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem name: combined-ca-bundle readOnly: true @@ -100,13 +88,20 @@ spec: name: public-tls-certs readOnly: true subPath: tls.key + - mountPath: /var/lib/config-data/default + name: config-data + readOnly: true + - mountPath: /var/lib/kolla/config_files/config.json + name: config-data + readOnly: true + subPath: glance-httpd-config.json name: glance-httpd - args: - --single-child - -- - /bin/bash - -c - - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start + - /usr/local/bin/kolla_start volumeMounts: - mountPath: /var/lib/config-data/default name: config-data diff --git a/test/kuttl/tests/glance_split/01-assert.yaml b/test/kuttl/tests/glance_split/01-assert.yaml index 676ac6ca..3faf4fd9 100644 --- a/test/kuttl/tests/glance_split/01-assert.yaml +++ b/test/kuttl/tests/glance_split/01-assert.yaml @@ -81,7 +81,7 @@ spec: - -- - /bin/bash - -c - - /usr/sbin/httpd -DFOREGROUND + - /usr/local/bin/kolla_start command: - /usr/bin/dumb-init name: glance-httpd @@ -90,7 +90,7 @@ spec: - -- - /bin/bash - -c - - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start + - /usr/local/bin/kolla_start command: - /usr/bin/dumb-init name: glance-api @@ -129,7 +129,7 @@ spec: - -- - /bin/bash - -c - - /usr/sbin/httpd -DFOREGROUND + - /usr/local/bin/kolla_start command: - /usr/bin/dumb-init name: glance-httpd @@ -138,7 +138,7 @@ spec: - -- - /bin/bash - -c - - /usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start + - /usr/local/bin/kolla_start command: - /usr/bin/dumb-init name: glance-api