From c803892cef822b4392d1f21c863f5e0897f76d44 Mon Sep 17 00:00:00 2001 From: Brent Eagles Date: Tue, 5 Nov 2024 01:08:31 +0000 Subject: [PATCH] Implement predictable IPs for mDNS and bind9 This PR adds support for mounting generated predictable IPs and employing them in the relevant pods. --- ....openstack.org_designatebackendbind9s.yaml | 3 + ...signate.openstack.org_designatemdnses.yaml | 3 + .../designate.openstack.org_designates.yaml | 6 ++ api/v1beta1/common_types.go | 2 + api/v1beta1/designate_types.go | 1 + api/v1beta1/designate_webhook.go | 7 ++ api/v1beta1/designatebackendbind9_types.go | 4 + api/v1beta1/designatemdns_types.go | 4 + ....openstack.org_designatebackendbind9s.yaml | 3 + ...signate.openstack.org_designatemdnses.yaml | 3 + .../designate.openstack.org_designates.yaml | 6 ++ config/default/manager_default_images.yaml | 2 + controllers/designate_controller.go | 13 ++++ .../designatebackendbind9_controller.go | 74 +++++++++++++++++-- controllers/designatemdns_controller.go | 42 ++++++++--- pkg/designate/const.go | 9 +++ pkg/designate/initcontainer.go | 29 ++++++++ pkg/designate/predipcontainer.go | 55 ++++++++++++++ pkg/designatebackendbind9/deployment.go | 34 +++++---- pkg/designatebackendbind9/volumes.go | 26 +++++++ pkg/designatemdns/statefulset.go | 36 +++++---- pkg/designatemdns/volumes.go | 17 +++-- templates/common/setipalias.py | 62 ++++++++++++++++ templates/designate/bin/setipalias.sh | 18 +++++ .../designatebackendbind9/bin/setipalias.sh | 18 +++++ templates/designatemdns/bin/init.sh | 25 +++++++ templates/designatemdns/bin/setipalias.sh | 22 ++++++ .../config/designate-mdns-config.json | 8 +- templates/designatemdns/config/designate.conf | 39 ++++++++++ 29 files changed, 511 insertions(+), 60 deletions(-) create mode 100644 pkg/designate/predipcontainer.go create mode 100755 templates/common/setipalias.py create mode 100755 templates/designate/bin/setipalias.sh create mode 100755 templates/designatebackendbind9/bin/setipalias.sh create mode 100755 templates/designatemdns/bin/init.sh create mode 100755 templates/designatemdns/bin/setipalias.sh rename templates/{designate => designatemdns}/config/designate-mdns-config.json (54%) create mode 100644 templates/designatemdns/config/designate.conf diff --git a/api/bases/designate.openstack.org_designatebackendbind9s.yaml b/api/bases/designate.openstack.org_designatebackendbind9s.yaml index c613738c..5ffc978e 100644 --- a/api/bases/designate.openstack.org_designatebackendbind9s.yaml +++ b/api/bases/designate.openstack.org_designatebackendbind9s.yaml @@ -109,6 +109,9 @@ spec: But can also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement type: object + netUtilsImage: + description: NetUtilsImage - NetUtils container image + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network diff --git a/api/bases/designate.openstack.org_designatemdnses.yaml b/api/bases/designate.openstack.org_designatemdnses.yaml index f1ce7b75..ee8460af 100644 --- a/api/bases/designate.openstack.org_designatemdnses.yaml +++ b/api/bases/designate.openstack.org_designatemdnses.yaml @@ -109,6 +109,9 @@ spec: But can also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement type: object + netUtilsImage: + description: NetUtilsImage - NetUtils container image + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network diff --git a/api/bases/designate.openstack.org_designates.yaml b/api/bases/designate.openstack.org_designates.yaml index e8c02b35..34cbe347 100644 --- a/api/bases/designate.openstack.org_designates.yaml +++ b/api/bases/designate.openstack.org_designates.yaml @@ -515,6 +515,9 @@ spec: But can also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement type: object + netUtilsImage: + description: NetUtilsImage - NetUtils container image + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network @@ -866,6 +869,9 @@ spec: But can also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement type: object + netUtilsImage: + description: NetUtilsImage - NetUtils container image + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index 786ebee5..08b90e6e 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -37,6 +37,8 @@ const ( DesignateUnboundContainerImage = "quay.io/podified-antelope-centos9/openstack-unbound:current-podified" // DesignateBackendbind9ContainerImage is the fall-back container image for DesignateUnbound DesignateBackendbind9ContainerImage = "quay.io/podified-antelope-centos9/openstack-designate-backend-bind9:current-podified" + // NetUtilsContainerImage is the container image containing support for predictable IP pod injection + NetUtilsContainerImage = "quay.io/podified-antelope-centos9/openstack-netutils:current-podified" ) // DesignateTemplate defines common input parameters used by all Designate services diff --git a/api/v1beta1/designate_types.go b/api/v1beta1/designate_types.go index 0da1744a..7a607588 100644 --- a/api/v1beta1/designate_types.go +++ b/api/v1beta1/designate_types.go @@ -297,6 +297,7 @@ func SetupDefaults() { WorkerContainerImageURL: util.GetEnvVar("RELATED_IMAGE_DESIGNATE_WORKER_IMAGE_URL_DEFAULT", DesignateWorkerContainerImage), UnboundContainerImageURL: util.GetEnvVar("RELATED_IMAGE_DESIGNATE_UNBOUND_IMAGE_URL_DEFAULT", DesignateUnboundContainerImage), Backendbind9ContainerImageURL: util.GetEnvVar("RELATED_IMAGE_DESIGNATE_BACKENDBIND9_IMAGE_URL_DEFAULT", DesignateBackendbind9ContainerImage), + NetUtilsURL: util.GetEnvVar("RELATED_IMAGE_NETUTILS_IMAGE_URL_DEFAULT", NetUtilsContainerImage), DesignateAPIRouteTimeout: APITimeout, } diff --git a/api/v1beta1/designate_webhook.go b/api/v1beta1/designate_webhook.go index ddda8b11..1f5ffa7d 100644 --- a/api/v1beta1/designate_webhook.go +++ b/api/v1beta1/designate_webhook.go @@ -44,6 +44,7 @@ type DesignateDefaults struct { WorkerContainerImageURL string Backendbind9ContainerImageURL string UnboundContainerImageURL string + NetUtilsURL string DesignateAPIRouteTimeout int } @@ -86,6 +87,9 @@ func (spec *DesignateSpec) Default() { if spec.DesignateMdns.ContainerImage == "" { spec.DesignateMdns.ContainerImage = designateDefaults.MdnsContainerImageURL } + if spec.DesignateMdns.NetUtilsImage == "" { + spec.DesignateMdns.NetUtilsImage = designateDefaults.NetUtilsURL + } if spec.DesignateProducer.ContainerImage == "" { spec.DesignateProducer.ContainerImage = designateDefaults.ProducerContainerImageURL } @@ -95,6 +99,9 @@ func (spec *DesignateSpec) Default() { if spec.DesignateBackendbind9.ContainerImage == "" { spec.DesignateBackendbind9.ContainerImage = designateDefaults.Backendbind9ContainerImageURL } + if spec.DesignateBackendbind9.NetUtilsImage == "" { + spec.DesignateBackendbind9.NetUtilsImage = designateDefaults.NetUtilsURL + } if spec.DesignateUnbound.ContainerImage == "" { spec.DesignateUnbound.ContainerImage = designateDefaults.UnboundContainerImageURL } diff --git a/api/v1beta1/designatebackendbind9_types.go b/api/v1beta1/designatebackendbind9_types.go index 16183589..965ffbcf 100644 --- a/api/v1beta1/designatebackendbind9_types.go +++ b/api/v1beta1/designatebackendbind9_types.go @@ -78,6 +78,10 @@ type DesignateBackendbind9SpecBase struct { // +kubebuilder:validation:Optional // StorageRequest StorageRequest string `json:"storageRequest"` + + // +kubebuilder:validation:Optional + // NetUtilsImage - NetUtils container image + NetUtilsImage string `json:"netUtilsImage"` } // DesignateBackendbind9Status defines the observed state of DesignateBackendbind9 diff --git a/api/v1beta1/designatemdns_types.go b/api/v1beta1/designatemdns_types.go index 8b4e31be..36f2ed4d 100644 --- a/api/v1beta1/designatemdns_types.go +++ b/api/v1beta1/designatemdns_types.go @@ -72,6 +72,10 @@ type DesignateMdnsSpecBase struct { // +kubebuilder:validation:Optional // ControlNetworkName - specify which network attachment is to be used for control, notifys and zone transfers. ControlNetworkName string `json:"controlNetworkName"` + + // +kubebuilder:validation:Optional + // NetUtilsImage - NetUtils container image + NetUtilsImage string `json:"netUtilsImage"` } // DesignateMdnsStatus defines the observed state of DesignateMdns diff --git a/config/crd/bases/designate.openstack.org_designatebackendbind9s.yaml b/config/crd/bases/designate.openstack.org_designatebackendbind9s.yaml index c613738c..5ffc978e 100644 --- a/config/crd/bases/designate.openstack.org_designatebackendbind9s.yaml +++ b/config/crd/bases/designate.openstack.org_designatebackendbind9s.yaml @@ -109,6 +109,9 @@ spec: But can also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement type: object + netUtilsImage: + description: NetUtilsImage - NetUtils container image + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network diff --git a/config/crd/bases/designate.openstack.org_designatemdnses.yaml b/config/crd/bases/designate.openstack.org_designatemdnses.yaml index f1ce7b75..ee8460af 100644 --- a/config/crd/bases/designate.openstack.org_designatemdnses.yaml +++ b/config/crd/bases/designate.openstack.org_designatemdnses.yaml @@ -109,6 +109,9 @@ spec: But can also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement type: object + netUtilsImage: + description: NetUtilsImage - NetUtils container image + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network diff --git a/config/crd/bases/designate.openstack.org_designates.yaml b/config/crd/bases/designate.openstack.org_designates.yaml index e8c02b35..34cbe347 100644 --- a/config/crd/bases/designate.openstack.org_designates.yaml +++ b/config/crd/bases/designate.openstack.org_designates.yaml @@ -515,6 +515,9 @@ spec: But can also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement type: object + netUtilsImage: + description: NetUtilsImage - NetUtils container image + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network @@ -866,6 +869,9 @@ spec: But can also be used to add additional files. Those get added to the service config dir in /etc/ . TODO: -> implement type: object + netUtilsImage: + description: NetUtilsImage - NetUtils container image + type: string networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network diff --git a/config/default/manager_default_images.yaml b/config/default/manager_default_images.yaml index e6ad99dd..1251dd89 100644 --- a/config/default/manager_default_images.yaml +++ b/config/default/manager_default_images.yaml @@ -25,3 +25,5 @@ spec: value: quay.io/podified-antelope-centos9/openstack-designate-backend-bind9:current-podified - name: RELATED_IMAGE_DESIGNATE_UNBOUND_IMAGE_URL_DEFAULT value: quay.io/podified-antelope-centos9/openstack-unbound:current-podified + - name: RELATED_IMAGE_NETUTILS_IMAGE_URL_DEFAULT + value: quay.io/podified-antelope-centos9/openstack-netutils:current-podified diff --git a/controllers/designate_controller.go b/controllers/designate_controller.go index 0d289f9c..4a734e65 100644 --- a/controllers/designate_controller.go +++ b/controllers/designate_controller.go @@ -775,7 +775,11 @@ func (r *DesignateReconciler) reconcileNormal(ctx context.Context, instance *des } // Handle Mdns predictable IPs configmap + // We cannot have 0 mDNS pods so even though the CRD validation allows 0, don't allow it. mdnsReplicaCount := int(*instance.Spec.DesignateMdns.Replicas) + if mdnsReplicaCount < 1 { + mdnsReplicaCount = 1 + } var mdnsNames []string for i := 0; i < mdnsReplicaCount; i++ { mdnsNames = append(mdnsNames, fmt.Sprintf("mdns_address_%d", i)) @@ -798,6 +802,9 @@ func (r *DesignateReconciler) reconcileNormal(ctx context.Context, instance *des } // Handle Bind predictable IPs configmap + // Unlike mDNS, we can have 0 binds when byob is used. + // NOTE(beagles) Really it might make more sense to have BYOB be an explicit flag and not assume that a 0 + // value is a byob case. Something to think about. bindReplicaCount := int(*instance.Spec.DesignateBackendbind9.Replicas) var bindNames []string for i := 0; i < bindReplicaCount; i++ { @@ -1586,6 +1593,11 @@ func (r *DesignateReconciler) mdnsStatefulSetCreateOrUpdate(ctx context.Context, instance.Spec.DesignateMdns.NodeSelector = instance.Spec.NodeSelector } + if int(*instance.Spec.DesignateMdns.Replicas) < 1 { + var minReplicas int32 = 1 + instance.Spec.DesignateMdns.Replicas = &minReplicas + } + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, statefulSet, func() error { statefulSet.Spec = instance.Spec.DesignateMdns // Add in transfers from umbrella Designate CR (this instance) spec @@ -1598,6 +1610,7 @@ func (r *DesignateReconciler) mdnsStatefulSetCreateOrUpdate(ctx context.Context, statefulSet.Spec.ServiceAccount = instance.RbacResourceName() statefulSet.Spec.TLS = instance.Spec.DesignateAPI.TLS.Ca statefulSet.Spec.NodeSelector = instance.Spec.DesignateMdns.NodeSelector + statefulSet.Spec.Replicas = instance.Spec.DesignateMdns.Replicas err := controllerutil.SetControllerReference(instance, statefulSet, r.Scheme) if err != nil { diff --git a/controllers/designatebackendbind9_controller.go b/controllers/designatebackendbind9_controller.go index 5ac47819..f6d2252f 100644 --- a/controllers/designatebackendbind9_controller.go +++ b/controllers/designatebackendbind9_controller.go @@ -2,7 +2,6 @@ Copyright 2022. Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 @@ -25,6 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,6 +39,7 @@ import ( designatebackendbind9 "github.com/openstack-k8s-operators/designate-operator/pkg/designatebackendbind9" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" @@ -388,6 +389,19 @@ func (r *DesignateBackendbind9Reconciler) reconcileNormal(ctx context.Context, i return ctrl.Result{}, nil } + bindIPsUpdated, err := r.hasMapChanged(ctx, helper, instance, designate.BindPredIPConfigMap, designate.BindPredictableIPHash) + if err != nil { + return ctrl.Result{}, err + } + rndcUpdate, err := r.hasSecretChanged(ctx, helper, instance, designate.DesignateBindKeySecret, designate.RndcHash) + if err != nil { + return ctrl.Result{}, err + } + if rndcUpdate || bindIPsUpdated { + // Predictable IPs and/or rndc keys have been updated, we need to update the statefulset. + return ctrl.Result{}, nil + } + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) // Create ConfigMaps and Secrets - end @@ -514,7 +528,6 @@ func (r *DesignateBackendbind9Reconciler) reconcileNormal(ctx context.Context, i condition.SeverityWarning, condition.NetworkAttachmentsReadyErrorMessage, err.Error())) - return ctrl.Result{}, err } @@ -711,12 +724,15 @@ func (r *DesignateBackendbind9Reconciler) generateServiceConfigMaps( cms := []util.Template{ // ScriptsConfigMap { - Name: fmt.Sprintf("%s-scripts", instance.Name), - Namespace: instance.Namespace, - Type: util.TemplateTypeScripts, - InstanceType: instance.Kind, - AdditionalTemplate: map[string]string{"common.sh": "/common/common.sh"}, - Labels: cmLabels, + Name: fmt.Sprintf("%s-scripts", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeScripts, + InstanceType: instance.Kind, + AdditionalTemplate: map[string]string{ + "common.sh": "/common/common.sh", + "setipalias.py": "/common/setipalias.py", + }, + Labels: cmLabels, }, // Custom ConfigMap { @@ -762,3 +778,45 @@ func (r *DesignateBackendbind9Reconciler) createHashOfInputHashes( } return hash, changed, nil } + +func (r *DesignateBackendbind9Reconciler) hasMapChanged( + ctx context.Context, + h *helper.Helper, + instance *designatev1beta1.DesignateBackendbind9, + mapName string, + hashKey string, +) (bool, error) { + configMap := &corev1.ConfigMap{} + err := h.GetClient().Get(ctx, types.NamespacedName{Name: mapName, Namespace: instance.GetNamespace()}, configMap) + if err != nil { + r.GetLogger().Error(err, fmt.Sprintf("Unable to check config map %s for changes", mapName)) + return false, err + } + hashValue, err := configmap.Hash(configMap) + if err != nil { + return false, err + } + _, updated := util.SetHash(instance.Status.Hash, hashKey, hashValue) + return updated, nil +} + +func (r *DesignateBackendbind9Reconciler) hasSecretChanged( + ctx context.Context, + h *helper.Helper, + instance *designatev1beta1.DesignateBackendbind9, + secretName string, + hashKey string, +) (bool, error) { + found := &corev1.Secret{} + err := h.GetClient().Get(ctx, types.NamespacedName{Name: secretName, Namespace: instance.GetNamespace()}, found) + if err != nil { + r.GetLogger().Error(err, fmt.Sprintf("Unable to check secret %s for changes", secretName)) + return false, err + } + hashValue, err := secret.Hash(found) + if err != nil { + return false, err + } + _, updated := util.SetHash(instance.Status.Hash, hashKey, hashValue) + return updated, nil +} diff --git a/controllers/designatemdns_controller.go b/controllers/designatemdns_controller.go index f7d8c6c8..34d180f9 100644 --- a/controllers/designatemdns_controller.go +++ b/controllers/designatemdns_controller.go @@ -48,7 +48,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" - "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" @@ -674,7 +674,7 @@ func (r *DesignateMdnsReconciler) getSecret( envVars *map[string]env.Setter, prefix string, ) (ctrl.Result, error) { - secret, hash, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + secret, hash, err := oko_secret.GetSecret(ctx, h, secretName, instance.Namespace) if err != nil { if k8s_errors.IsNotFound(err) { h.GetLogger().Info(fmt.Sprintf("Secret %s not found", secretName)) @@ -824,15 +824,39 @@ func (r *DesignateMdnsReconciler) generateServiceConfigMaps( } templateParameters["AllowCIDR"] = cidr + transportURLSecret, _, err := oko_secret.GetSecret(ctx, h, instance.Spec.TransportURLSecret, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + r.GetLogger().Info(fmt.Sprintf("TransportURL secret %s not found", instance.Spec.TransportURLSecret)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return err + } + templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"]) + cms := []util.Template{ // ScriptsConfigMap { - Name: fmt.Sprintf("%s-scripts", instance.Name), - Namespace: instance.Namespace, - Type: util.TemplateTypeScripts, - InstanceType: instance.Kind, - AdditionalTemplate: map[string]string{"common.sh": "/common/common.sh"}, - Labels: cmLabels, + Name: fmt.Sprintf("%s-scripts", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeScripts, + InstanceType: instance.Kind, + AdditionalTemplate: map[string]string{ + "common.sh": "/common/common.sh", + "setipalias.py": "/common/setipalias.py", + }, + Labels: cmLabels, }, // Custom ConfigMap { @@ -846,7 +870,7 @@ func (r *DesignateMdnsReconciler) generateServiceConfigMaps( }, } - return secret.EnsureSecrets(ctx, h, instance, cms, envVars) + return oko_secret.EnsureSecrets(ctx, h, instance, cms, envVars) } // createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart diff --git a/pkg/designate/const.go b/pkg/designate/const.go index e619aa1b..935f7e69 100644 --- a/pkg/designate/const.go +++ b/pkg/designate/const.go @@ -52,4 +52,13 @@ const ( PoolsYamlPath = "templates/designatepoolmanager/config/pools.yaml.tmpl" PoolsYamlHash = "pools-yaml-hash" + + // BindPredictableIPHash key for status hash + BindPredictableIPHash = "Bind IP Map" + + // RndcHash key for status hash + RndcHash = "Rndc keys" + + // PredictableIPCommand - + PredictableIPCommand = "/usr/local/bin/container-scripts/setipalias.sh" ) diff --git a/pkg/designate/initcontainer.go b/pkg/designate/initcontainer.go index e2afdb93..de2b7402 100644 --- a/pkg/designate/initcontainer.go +++ b/pkg/designate/initcontainer.go @@ -31,11 +31,40 @@ type APIDetails struct { Privileged bool } +type InitContainerDetails struct { + ContainerImage string + VolumeMounts []corev1.VolumeMount + EnvVars []corev1.EnvVar +} + const ( // InitContainerCommand - InitContainerCommand = "/usr/local/bin/container-scripts/init.sh" ) +func SimpleInitContainer(init InitContainerDetails) corev1.Container { + runAsUser := int64(0) + + args := []string{ + "-c", + InitContainerCommand, + } + + return corev1.Container{ + Name: "init", + Image: init.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Command: []string{ + "/bin/bash", + }, + Args: args, + Env: init.EnvVars, + VolumeMounts: init.VolumeMounts, + } +} + // InitContainer - init container for designate api pods func InitContainer(init APIDetails) []corev1.Container { runAsUser := int64(0) diff --git a/pkg/designate/predipcontainer.go b/pkg/designate/predipcontainer.go new file mode 100644 index 00000000..38f5f851 --- /dev/null +++ b/pkg/designate/predipcontainer.go @@ -0,0 +1,55 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package designate + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" +) + +type PredIPContainerDetails struct { + ContainerImage string + VolumeMounts []corev1.VolumeMount + Command string + EnvVars []corev1.EnvVar +} + +func PredictableIPContainer(init PredIPContainerDetails) corev1.Container { + + args := []string{ + "-c", + init.Command, + } + + capabilities := []corev1.Capability{"NET_ADMIN", "SYS_ADMIN", "SYS_NICE"} + return corev1.Container{ + Name: "predictableips", + Image: init.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: capabilities, + Drop: []corev1.Capability{}, + }, + RunAsUser: ptr.To(int64(0)), + }, + Command: []string{ + "/bin/bash", + }, + Args: args, + Env: init.EnvVars, + VolumeMounts: init.VolumeMounts, + } +} diff --git a/pkg/designatebackendbind9/deployment.go b/pkg/designatebackendbind9/deployment.go index 70d92c0a..49d8f3e7 100644 --- a/pkg/designatebackendbind9/deployment.go +++ b/pkg/designatebackendbind9/deployment.go @@ -162,24 +162,28 @@ func StatefulSet( // TODO: bind's init container doesn't need most of this stuff. It doesn't use rabbitmq, redis or access the // database. Should clean this up! - initContainerDetails := designate.APIDetails{ - ContainerImage: instance.Spec.ContainerImage, - DatabaseHost: instance.Spec.DatabaseHostname, - DatabaseName: designate.DatabaseName, - OSPSecret: instance.Spec.Secret, - TransportURLSecret: instance.Spec.TransportURLSecret, - UserPasswordSelector: instance.Spec.PasswordSelectors.Service, - VolumeMounts: getInitVolumeMounts(), - } - statefulSet.Spec.Template.Spec.InitContainers = designate.InitContainer(initContainerDetails) - - // TODO: Clean up this hack - // Add custom config for the API Service envVars = map[string]env.Setter{} - envVars["CustomConf"] = env.SetValue(common.CustomServiceConfigFileName) envVars["POD_NAME"] = env.DownwardAPI("metadata.name") + envVars["CustomConf"] = env.SetValue(common.CustomServiceConfigFileName) + envVars["MAP_PREFIX"] = env.SetValue("bind_address_") envVars["RNDC_PREFIX"] = env.SetValue(designate.DesignateRndcKey) - statefulSet.Spec.Template.Spec.InitContainers[0].Env = env.MergeEnvs(statefulSet.Spec.Template.Spec.InitContainers[0].Env, envVars) + env := env.MergeEnvs([]corev1.EnvVar{}, envVars) + initContainerDetails := designate.InitContainerDetails{ + ContainerImage: instance.Spec.ContainerImage, + VolumeMounts: getInitVolumeMounts(), + EnvVars: env, + } + predIPContainerDetails := designate.PredIPContainerDetails{ + ContainerImage: instance.Spec.NetUtilsImage, + VolumeMounts: getPredIPVolumeMounts(), + EnvVars: env, + Command: designate.PredictableIPCommand, + } + + statefulSet.Spec.Template.Spec.InitContainers = []corev1.Container{ + designate.SimpleInitContainer(initContainerDetails), + designate.PredictableIPContainer(predIPContainerDetails), + } return statefulSet } diff --git a/pkg/designatebackendbind9/volumes.go b/pkg/designatebackendbind9/volumes.go index 984d181b..2680502c 100644 --- a/pkg/designatebackendbind9/volumes.go +++ b/pkg/designatebackendbind9/volumes.go @@ -28,6 +28,7 @@ const ( mergedConfigVolume = "designatebackendbind9-config-data-merged" logVolume = "designatebackendbind9-log-volume" rndcKeys = "designatebackendbind9-keys" + bindIPs = "designate-bind-ips" ) // NOTE(beagles): I vacillated on using designate.GetVolumes() here and appending the extra entries and may still. There @@ -86,6 +87,17 @@ func getServicePodVolumes(baseConfigMapName string) []corev1.Volume { }, }, }, + { + Name: bindIPs, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &configMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: designate.BindPredIPConfigMap, + }, + }, + }, + }, } } @@ -123,6 +135,20 @@ func getInitVolumeMounts() []corev1.VolumeMount { } } +func getPredIPVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: scriptVolume, + MountPath: "/usr/local/bin/container-scripts", + ReadOnly: true, + }, + { + Name: bindIPs, + MountPath: "/var/lib/predictableips", + }, + } +} + func getServicePodVolumeMounts(persistentData string) []corev1.VolumeMount { return []corev1.VolumeMount{ { diff --git a/pkg/designatemdns/statefulset.go b/pkg/designatemdns/statefulset.go index 16dcc5f0..0e2d1969 100644 --- a/pkg/designatemdns/statefulset.go +++ b/pkg/designatemdns/statefulset.go @@ -16,8 +16,6 @@ limitations under the License. package designatemdns import ( - "fmt" - designatev1beta1 "github.com/openstack-k8s-operators/designate-operator/api/v1beta1" designate "github.com/openstack-k8s-operators/designate-operator/pkg/designate" common "github.com/openstack-k8s-operators/lib-common/modules/common" @@ -43,9 +41,9 @@ func StatefulSet( annotations map[string]string, ) *appsv1.StatefulSet { rootUser := int64(0) - serviceName := fmt.Sprintf("%s-mdns", designate.ServiceName) - volumes := GetVolumes(designate.GetOwningDesignateName(instance)) - volumeMounts := GetVolumeMounts(serviceName) + serviceName := instance.Name + volumes := GetVolumes(serviceName) + volumeMounts := designate.GetVolumeMounts(serviceName) livenessProbe := &corev1.Probe{ // TODO might need tuning @@ -133,16 +131,26 @@ func StatefulSet( statefulSet.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector } - initContainerDetails := designate.APIDetails{ - ContainerImage: instance.Spec.ContainerImage, - DatabaseHost: instance.Spec.DatabaseHostname, - DatabaseName: designate.DatabaseName, - OSPSecret: instance.Spec.Secret, - TransportURLSecret: instance.Spec.TransportURLSecret, - UserPasswordSelector: instance.Spec.PasswordSelectors.Service, - VolumeMounts: designate.GetInitVolumeMounts(), + envVars = map[string]env.Setter{} + envVars["POD_NAME"] = env.DownwardAPI("metadata.name") + envVars["MAP_PREFIX"] = env.SetValue("mdns_address_") + podEnv := env.MergeEnvs([]corev1.EnvVar{}, envVars) + initContainerDetails := designate.InitContainerDetails{ + ContainerImage: instance.Spec.ContainerImage, + VolumeMounts: designate.GetInitVolumeMounts(), + EnvVars: podEnv, + } + predIPContainerDetails := designate.PredIPContainerDetails{ + ContainerImage: instance.Spec.NetUtilsImage, + VolumeMounts: getPredIPVolumeMounts(), + EnvVars: podEnv, + Command: designate.PredictableIPCommand, + } + + statefulSet.Spec.Template.Spec.InitContainers = []corev1.Container{ + designate.SimpleInitContainer(initContainerDetails), + designate.PredictableIPContainer(predIPContainerDetails), } - statefulSet.Spec.Template.Spec.InitContainers = designate.InitContainer(initContainerDetails) return statefulSet } diff --git a/pkg/designatemdns/volumes.go b/pkg/designatemdns/volumes.go index ef40f5c3..3f690bbb 100644 --- a/pkg/designatemdns/volumes.go +++ b/pkg/designatemdns/volumes.go @@ -37,13 +37,16 @@ func GetVolumes(name string) []corev1.Volume { ) } -func GetVolumeMounts(serviceName string) []corev1.VolumeMount { - return append( - designate.GetVolumeMounts(serviceName), - corev1.VolumeMount{ - Name: "bind-ips", - MountPath: "/var/lib/bind-ips", +func getPredIPVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: "scripts", + MountPath: "/usr/local/bin/container-scripts", ReadOnly: true, }, - ) + { + Name: "bind-ips", + MountPath: "/var/lib/predictableips", + }, + } } diff --git a/templates/common/setipalias.py b/templates/common/setipalias.py new file mode 100755 index 00000000..5fbecbb6 --- /dev/null +++ b/templates/common/setipalias.py @@ -0,0 +1,62 @@ +#!/usr/bin/python3 +import os +import sys +import ipaddress +import netifaces +from pyroute2 import IPRoute + +mapping_prefix = os.environ.get("MAP_PREFIX") +if not len(mapping_prefix): + print(f"{sys.argv[0]} requires MAP_PREFIX to be set in environment variables") + sys.exit(1) + +print(f"Using {mapping_prefix} as the map prefix") + +nodefile = "" +podname = os.environ.get("POD_NAME").strip() +if not len(podname): + print(f"{sys.argv[0]} requires POD_NAME in environment variables") + sys.exit(1) + +print(f"Pod name is {podname}") +namepieces = podname.split('-') +pod_index = namepieces[-1] +nodefile = f"{mapping_prefix}{pod_index}" + +interface_name = os.environ.get("NAD_NAME", "designate").strip() + +print(f"working with address file {nodefile}") +filename = os.path.join('/var/lib/predictableips', nodefile) +if not os.path.exists(filename): + print(f"Required alias address file {filename} does not exist") + sys.exit(1) + +ip = IPRoute() +designateinterface = ip.link_lookup(ifname=interface_name) + +if not len(designateinterface): + print(f"{interface_name} attachment not present") + sys.exit(1) + + +ipfile = open(filename, "r") +ipaddr = ipfile.read() +ipfile.close() + +if ipaddr: + print(f"Setting {ipaddr} on {interface_name}") + # Get our current addresses so we can avoid trying to set the + # same address again. + version = ipaddress.ip_address(ipaddr).version + ifaceinfo = netifaces.ifaddresses(interface_name)[ + netifaces.AF_INET if version == 4 else netifaces.AF_INET6] + current_addresses = [x['addr'] for x in ifaceinfo] + if ipaddr not in current_addresses: + mask_value = 32 + if version == 6: + mask_value = 128 + ip.addr('add', index = designateinterface[0], address=ipaddr, mask=mask_value) +else: + print('No IP address found') + +ip.close() diff --git a/templates/designate/bin/setipalias.sh b/templates/designate/bin/setipalias.sh new file mode 100755 index 00000000..c4d2d75e --- /dev/null +++ b/templates/designate/bin/setipalias.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Copyright 2024 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -ex + +/usr/local/bin/container-scripts/setipalias.py diff --git a/templates/designatebackendbind9/bin/setipalias.sh b/templates/designatebackendbind9/bin/setipalias.sh new file mode 100755 index 00000000..c4d2d75e --- /dev/null +++ b/templates/designatebackendbind9/bin/setipalias.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Copyright 2024 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -ex + +/usr/local/bin/container-scripts/setipalias.py diff --git a/templates/designatemdns/bin/init.sh b/templates/designatemdns/bin/init.sh new file mode 100755 index 00000000..c8fbbd5a --- /dev/null +++ b/templates/designatemdns/bin/init.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2024 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -ex + +# expect that the common.sh is in the same dir as the calling script +SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +. ${SCRIPTPATH}/common.sh --source-only + +# Merge all templates from config CM +for dir in /var/lib/config-data/default; do + merge_config_dir ${dir} +done diff --git a/templates/designatemdns/bin/setipalias.sh b/templates/designatemdns/bin/setipalias.sh new file mode 100755 index 00000000..bc831117 --- /dev/null +++ b/templates/designatemdns/bin/setipalias.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Copyright 2024 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -ex + +# expect that the common.sh is in the same dir as the calling script +# +SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +/usr/local/bin/container-scripts/setipalias.py diff --git a/templates/designate/config/designate-mdns-config.json b/templates/designatemdns/config/designate-mdns-config.json similarity index 54% rename from templates/designate/config/designate-mdns-config.json rename to templates/designatemdns/config/designate-mdns-config.json index 320506bf..a36dc78b 100644 --- a/templates/designate/config/designate-mdns-config.json +++ b/templates/designatemdns/config/designate-mdns-config.json @@ -1,5 +1,5 @@ { - "command": "/usr/bin/designate-mdns --config-file /etc/designate/designate.conf --config-dir /etc/designate/designate.conf.d", + "command": "/usr/bin/designate-mdns --config-dir /etc/designate/designate.conf", "config_files": [ { "source": "/var/lib/config-data/merged/designate.conf", @@ -7,12 +7,6 @@ "owner": "root:designate", "perm": "0750" }, - { - "source": "/var/lib/config-data/merged/custom.conf", - "dest": "/etc/designate/designate.conf.d/custom.conf", - "owner": "designate", - "perm": "0600" - }, { "source": "/var/lib/config-data/merged/my.cnf", "dest": "/etc/my.cnf", diff --git a/templates/designatemdns/config/designate.conf b/templates/designatemdns/config/designate.conf new file mode 100644 index 00000000..f330ae32 --- /dev/null +++ b/templates/designatemdns/config/designate.conf @@ -0,0 +1,39 @@ +[DEFAULT] +rpc_response_timeout=60 +quota_api_export_size=1000 +quota_recordset_records=20 +quota_zone_records=500 +quota_zone_recordsets=500 +quota_zones=10 +root-helper=sudo +state_path=/etc/designate/data +transport_url={{ .TransportURL }} + +[database] +connection={{ .DatabaseConnection }} + +[storage:sqlalchemy] +connection={{ .DatabaseConnection }} + +[service:api] +quotas_verify_project_id=True +auth_strategy=keystone +enable_api_admin=True +enable_api_v2=True +enable_host_header=True +enabled_extensions_admin=quotas + +[service:mdns] +workers=2 +listen=0.0.0.0:5354 + +[oslo_messaging_notifications] +topics=notifications +driver=messagingv2 + +[oslo_concurrency] +lock_path=/opt/stack/data/designate + +[oslo_policy] +enforce_scope=True +enforce_new_defaults=True