diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e759b12d..8eebcfa7 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - uses: golangci/golangci-lint-action@v3 with: args: --timeout=3m @@ -29,7 +29,7 @@ jobs: - name: Setup Go environment uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - uses: actions/setup-python@v4 - name: Run pre-commit checks on changes files uses: pre-commit/action@v3.0.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index ccc9e2c4..cfbabe4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add image configuration for CSI sidecars. - Check kernel module parameters for DRBD on load. - Automatically set SELinux labels when loading kernel modules. +- Allow more complex node selection by adding `LinstorCluster.spec.nodeAffinity`. ### Changed diff --git a/Dockerfile b/Dockerfile index 45e406d5..f1dbc43a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.4 # Build the manager binary -FROM --platform=$BUILDPLATFORM golang:1.20 as builder +FROM --platform=$BUILDPLATFORM golang:1.21 as builder WORKDIR /workspace # Copy the Go Modules manifests diff --git a/api/v1/linstorcluster_types.go b/api/v1/linstorcluster_types.go index 891bf5ce..13589853 100644 --- a/api/v1/linstorcluster_types.go +++ b/api/v1/linstorcluster_types.go @@ -18,6 +18,7 @@ package v1 import ( cmmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -38,6 +39,11 @@ type LinstorClusterSpec struct { // +kubebuilder:validation:Optional NodeSelector map[string]string `json:"nodeSelector,omitempty"` + // NodeAffinity selects the nodes on which LINSTOR Satellite will be deployed. + // See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + // +kubebuilder:validation:Optional + NodeAffinity *corev1.NodeSelector `json:"nodeAffinity,omitempty"` + // Properties to apply on the cluster level. // // Use to create default settings for DRBD that should apply to all resources or to configure some other cluster diff --git a/api/v1/linstorsatelliteconfiguration_types.go b/api/v1/linstorsatelliteconfiguration_types.go index af4292ee..2ee0009f 100644 --- a/api/v1/linstorsatelliteconfiguration_types.go +++ b/api/v1/linstorsatelliteconfiguration_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -30,6 +31,11 @@ type LinstorSatelliteConfigurationSpec struct { // +kubebuilder:validation:Optional NodeSelector map[string]string `json:"nodeSelector,omitempty"` + // NodeAffinity selects which LinstorSatellite resources this spec should be applied to. + // See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + // +kubebuilder:validation:Optional + NodeAffinity *corev1.NodeSelector `json:"nodeAffinity,omitempty"` + // Patches is a list of kustomize patches to apply. // // See https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/ for how to create patches. diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index eef02734..430b0701 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -22,6 +22,7 @@ package v1 import ( metav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" apismetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -140,6 +141,11 @@ func (in *LinstorClusterSpec) DeepCopyInto(out *LinstorClusterSpec) { (*out)[key] = val } } + if in.NodeAffinity != nil { + in, out := &in.NodeAffinity, &out.NodeAffinity + *out = new(corev1.NodeSelector) + (*in).DeepCopyInto(*out) + } if in.Properties != nil { in, out := &in.Properties, &out.Properties *out = make([]LinstorControllerProperty, len(*in)) @@ -485,6 +491,11 @@ func (in *LinstorSatelliteConfigurationSpec) DeepCopyInto(out *LinstorSatelliteC (*out)[key] = val } } + if in.NodeAffinity != nil { + in, out := &in.NodeAffinity, &out.NodeAffinity + *out = new(corev1.NodeSelector) + (*in).DeepCopyInto(*out) + } if in.Patches != nil { in, out := &in.Patches, &out.Patches *out = make([]Patch, len(*in)) diff --git a/charts/piraeus/templates/crds.yaml b/charts/piraeus/templates/crds.yaml index 9f11c78c..b343e941 100644 --- a/charts/piraeus/templates/crds.yaml +++ b/charts/piraeus/templates/crds.yaml @@ -129,6 +129,91 @@ spec: for accessing remotes for backups. See https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/#s-encrypt_commands for more information." type: string + nodeAffinity: + description: NodeAffinity selects the nodes on which LINSTOR Satellite + will be deployed. See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms + are ORed. + items: + description: A null or empty node selector term matches no objects. + The requirements of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's + labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's + fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic nodeSelector: additionalProperties: type: string @@ -569,6 +654,91 @@ spec: connections on behalf of DRBD." type: boolean type: object + nodeAffinity: + description: NodeAffinity selects which LinstorSatellite resources + this spec should be applied to. See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms + are ORed. + items: + description: A null or empty node selector term matches no objects. + The requirements of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's + labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's + fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic nodeSelector: additionalProperties: type: string diff --git a/config/crd/bases/piraeus.io_linstorclusters.yaml b/config/crd/bases/piraeus.io_linstorclusters.yaml index 047c6276..267961bc 100644 --- a/config/crd/bases/piraeus.io_linstorclusters.yaml +++ b/config/crd/bases/piraeus.io_linstorclusters.yaml @@ -128,6 +128,91 @@ spec: for accessing remotes for backups. See https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/#s-encrypt_commands for more information." type: string + nodeAffinity: + description: NodeAffinity selects the nodes on which LINSTOR Satellite + will be deployed. See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms + are ORed. + items: + description: A null or empty node selector term matches no objects. + The requirements of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's + labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's + fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic nodeSelector: additionalProperties: type: string diff --git a/config/crd/bases/piraeus.io_linstorsatelliteconfigurations.yaml b/config/crd/bases/piraeus.io_linstorsatelliteconfigurations.yaml index 0ae34f3a..3b634e31 100644 --- a/config/crd/bases/piraeus.io_linstorsatelliteconfigurations.yaml +++ b/config/crd/bases/piraeus.io_linstorsatelliteconfigurations.yaml @@ -73,6 +73,91 @@ spec: connections on behalf of DRBD." type: boolean type: object + nodeAffinity: + description: NodeAffinity selects which LinstorSatellite resources + this spec should be applied to. See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms + are ORed. + items: + description: A null or empty node selector term matches no objects. + The requirements of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's + labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's + fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic nodeSelector: additionalProperties: type: string diff --git a/docs/reference/linstorcluster.md b/docs/reference/linstorcluster.md index 70fb8c82..d4de4dc4 100644 --- a/docs/reference/linstorcluster.md +++ b/docs/reference/linstorcluster.md @@ -18,6 +18,9 @@ excluded by the selector will not be able to run any workload using a Piraeus vo If empty (the default), Piraeus will be deployed on all nodes in the cluster. +When this is used together with `.spec.nodeAffinity`, both need to match in order for +a node to run Piraeus. + #### Example This example restricts Piraeus Datastore to nodes matching `example.com/storage: "yes"`: @@ -32,6 +35,39 @@ spec: example.com/storage: "yes" ``` +### `.spec.nodeAffinity` + +Selects on which nodes Piraeus Datastore should be deployed. Nodes that are +excluded by the affinity will not be able to run any workload using a Piraeus volume. + +If empty (the default), Piraeus will be deployed on all nodes in the cluster. + +When this is used together with `.spec.nodeSelector`, both need to match in order for +a node to run Piraeus. + +#### Example + +This example restricts Piraeus Datastore to nodes in zones `a` and `b`, but not on `control-plane` nodes: + +```yaml +apiVersion: piraeus.io/v1 +kind: LinstorCluster +metadata: + name: linstorcluster +spec: + nodeAffinity: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - a + - b + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist +``` + + ### `.spec.repository` Sets the default image registry to use for all Piraeus images. The full image name is diff --git a/docs/reference/linstorsatelliteconfiguration.md b/docs/reference/linstorsatelliteconfiguration.md index e8a67d5a..d95cd150 100644 --- a/docs/reference/linstorsatelliteconfiguration.md +++ b/docs/reference/linstorsatelliteconfiguration.md @@ -27,6 +27,33 @@ spec: value: "no" ``` +### `.spec.nodeAffinity` + +Selects which nodes the LinstorSatelliteConfiguration should apply to. If empty, the configuration applies to all nodes. + +When this is used together with `.spec.nodeAffinity`, both need to match in order for the configuration to apply to a +node. + +#### Example + +This example sets the `AutoplaceTarget` property to `no` on all non-worker nodes: + +```yaml +apiVersion: piraeus.io/v1 +kind: LinstorSatelliteConfiguration +metadata: + name: disabled-nodes +spec: + nodeAffinity: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + properties: + - name: AutoplaceTarget + value: "no" +``` + ### `.spec.properties` Sets the given properties on the LINSTOR Satellite level. diff --git a/go.mod b/go.mod index 27c86fa9..1803e382 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/piraeusdatastore/piraeus-operator/v2 -go 1.20 +go 1.21 require ( github.com/LINBIT/golinstor v0.49.0 @@ -14,9 +14,10 @@ require ( golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/time v0.4.0 gonum.org/v1/gonum v0.14.0 - k8s.io/api v0.28.3 - k8s.io/apimachinery v0.28.3 - k8s.io/client-go v0.28.3 + k8s.io/api v0.28.4 + k8s.io/apimachinery v0.28.4 + k8s.io/client-go v0.28.4 + k8s.io/component-helpers v0.28.4 sigs.k8s.io/controller-runtime v0.16.3 sigs.k8s.io/kustomize/api v0.15.0 sigs.k8s.io/kustomize/kyaml v0.15.0 diff --git a/go.sum b/go.sum index bccaf99d..fa9e2576 100644 --- a/go.sum +++ b/go.sum @@ -245,16 +245,18 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= -k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= +k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY= +k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0= k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc= -k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= -k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= -k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= -k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= +k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8= +k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= +k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY= +k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4= k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= +k8s.io/component-helpers v0.28.4 h1:+X9VXT5+jUsRdC26JyMZ8Fjfln7mSjgumafocE509C4= +k8s.io/component-helpers v0.28.4/go.mod h1:8LzMalOQ0K10tkBJWBWq8h0HTI9HDPx4WT3QvTFn9Ro= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= diff --git a/internal/controller/linstorcluster_controller.go b/internal/controller/linstorcluster_controller.go index a138a610..32f7e614 100644 --- a/internal/controller/linstorcluster_controller.go +++ b/internal/controller/linstorcluster_controller.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + "slices" "sort" "strings" "time" @@ -36,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + schedulingcorev1 "k8s.io/component-helpers/scheduling/corev1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -147,6 +149,13 @@ func (r *LinstorClusterReconciler) reconcileAppliedResource(ctx context.Context, return err } + if lcluster.Spec.NodeAffinity != nil { + satelliteNodes.Items = slices.DeleteFunc(satelliteNodes.Items, func(node corev1.Node) bool { + matches, _ := schedulingcorev1.MatchNodeSelectorTerms(&node, lcluster.Spec.NodeAffinity) + return !matches + }) + } + satelliteConfigs := piraeusiov1.LinstorSatelliteConfigurationList{} err = r.Client.List(ctx, &satelliteConfigs) if err != nil { @@ -299,9 +308,22 @@ func (r *LinstorClusterReconciler) kustomizeControllerResources(lcluster *piraeu return resmap.New(), nil } - var patches []kusttypes.Patch resourceDirs := []string{"controller"} + patches, err := ClusterLinstorControllerNodeSelector(lcluster.Spec.NodeSelector) + if err != nil { + return nil, err + } + + if lcluster.Spec.NodeAffinity != nil { + p, err := ClusterLinstorControllerNodeAffinityPatch(lcluster.Spec.NodeAffinity) + if err != nil { + return nil, err + } + + patches = append(patches, p...) + } + if lcluster.Spec.LinstorPassphraseSecret != "" { p, err := ClusterLinstorPassphrasePatch(lcluster.Spec.LinstorPassphraseSecret) if err != nil { @@ -390,6 +412,13 @@ func (r *LinstorClusterReconciler) kustomizeCsiResources(lcluster *piraeusiov1.L return nil, err } + p, err := ClusterCSIControllerNodeSelector(lcluster.Spec.NodeSelector) + if err != nil { + return nil, err + } + + patches = append(patches, p...) + endpointPatches, err := ClusterApiEndpointPatch(LinstorControllerUrl(lcluster)) if err != nil { return nil, err @@ -397,6 +426,21 @@ func (r *LinstorClusterReconciler) kustomizeCsiResources(lcluster *piraeusiov1.L patches = append(patches, endpointPatches...) + if lcluster.Spec.NodeAffinity != nil { + nodePatches, err := ClusterCSINodeNodeAffinityPatch(lcluster.Spec.NodeAffinity) + if err != nil { + return nil, err + } + + controllerPatches, err := ClusterCSIControllerNodeAffinityPatch(lcluster.Spec.NodeAffinity) + if err != nil { + return nil, err + } + + patches = append(patches, nodePatches...) + patches = append(patches, controllerPatches...) + } + if lcluster.Spec.ApiTLS != nil { controllerSecret := lcluster.Spec.ApiTLS.GetCsiControllerSecretName() nodeSecret := lcluster.Spec.ApiTLS.GetCsiNodeSecretName() @@ -444,6 +488,15 @@ func (r *LinstorClusterReconciler) kustomizeHAControllerResources(lcluster *pira return nil, err } + if lcluster.Spec.NodeAffinity != nil { + p, err := ClusterHAControllerNodeAffinityPatch(lcluster.Spec.NodeAffinity) + if err != nil { + return nil, err + } + + patches = append(patches, p...) + } + return r.kustomize([]string{"ha-controller"}, lcluster, imgs, patches...) } @@ -499,7 +552,7 @@ func (r *LinstorClusterReconciler) kustomizeLinstorSatellite(lcluster *piraeusio patches := []utils.JsonPatch{renamePatch, repositoryPatch, clusterRefPatch} - cfg := merge.SatelliteConfigurations(node.ObjectMeta.Labels, configs...) + cfg := merge.SatelliteConfigurations(node, configs...) if cfg.Spec.InternalTLS != nil { patches = append(patches, utils.JsonPatch{ diff --git a/internal/controller/linstorcluster_test.go b/internal/controller/linstorcluster_test.go index 01e95ba0..59970777 100644 --- a/internal/controller/linstorcluster_test.go +++ b/internal/controller/linstorcluster_test.go @@ -59,7 +59,10 @@ var _ = Describe("LinstorCluster controller", func() { Describe("with cluster nodes present", func() { BeforeEach(func(ctx context.Context) { err := k8sClient.Create(ctx, &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{Name: "node-1a", Labels: map[string]string{"topology.kubernetes.io/zone": "a"}}, + ObjectMeta: metav1.ObjectMeta{Name: "node-1a", Labels: map[string]string{ + "topology.kubernetes.io/zone": "a", + "example.com/exclude": "yes", + }}, }) Expect(err).NotTo(HaveOccurred()) @@ -241,6 +244,53 @@ var _ = Describe("LinstorCluster controller", func() { return controller.Image }, DefaultTimeout, DefaultCheckInterval).Should(HavePrefix("piraeus.io/test")) }) + + It("should apply affinity set on the cluster resource", func(ctx context.Context) { + Eventually(func() bool { + var satellites piraeusiov1.LinstorSatelliteList + err := k8sClient.List(ctx, &satellites) + Expect(err).NotTo(HaveOccurred()) + + return len(satellites.Items) == 3 + }, DefaultTimeout, DefaultCheckInterval).Should(BeTrue()) + + var cluster piraeusiov1.LinstorCluster + err := k8sClient.Get(ctx, types.NamespacedName{Name: "default"}, &cluster) + Expect(err).NotTo(HaveOccurred()) + + cluster.Spec.NodeAffinity = &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"b"}, + }, + { + Key: "example.com/exclude", + Operator: corev1.NodeSelectorOpDoesNotExist, + }, + }, + }}, + } + + err = k8sClient.Update(ctx, &cluster) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []string { + var satellites piraeusiov1.LinstorSatelliteList + err := k8sClient.List(ctx, &satellites) + Expect(err).NotTo(HaveOccurred()) + + var result []string + for i := range satellites.Items { + if satellites.Items[i].DeletionTimestamp == nil { + result = append(result, satellites.Items[i].Name) + } + } + return result + }, DefaultTimeout, DefaultCheckInterval).Should(ConsistOf("node-2a")) + }) }) }) diff --git a/internal/controller/patches.go b/internal/controller/patches.go index 5f58ea6d..1ce8e104 100644 --- a/internal/controller/patches.go +++ b/internal/controller/patches.go @@ -7,6 +7,7 @@ import ( "strings" cmmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" kusttypes "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/yaml" @@ -46,6 +47,42 @@ func ClusterLinstorInternalTLSCertManagerPatch(secretName string, issuer *cmmeta ) } +func ClusterLinstorControllerNodeSelector(selector map[string]string) ([]kusttypes.Patch, error) { + return render( + cluster.Resources, + "patches/linstor-controller-selector.yaml", + map[string]any{ + "NODE_SELECTOR": selector, + }) +} + +func ClusterLinstorControllerNodeAffinityPatch(affinity *corev1.NodeSelector) ([]kusttypes.Patch, error) { + return render( + cluster.Resources, + "patches/linstor-controller-node-affinity.yaml", + map[string]any{ + "NODE_AFFINITY": affinity, + }) +} + +func ClusterCSIControllerNodeSelector(selector map[string]string) ([]kusttypes.Patch, error) { + return render( + cluster.Resources, + "patches/csi-controller-selector.yaml", + map[string]any{ + "NODE_SELECTOR": selector, + }) +} + +func ClusterCSIControllerNodeAffinityPatch(affinity *corev1.NodeSelector) ([]kusttypes.Patch, error) { + return render( + cluster.Resources, + "patches/csi-controller-node-affinity.yaml", + map[string]any{ + "NODE_AFFINITY": affinity, + }) +} + func ClusterCSINodeSelectorPatch(selector map[string]string) ([]kusttypes.Patch, error) { return render( cluster.Resources, @@ -55,6 +92,15 @@ func ClusterCSINodeSelectorPatch(selector map[string]string) ([]kusttypes.Patch, }) } +func ClusterCSINodeNodeAffinityPatch(affinity *corev1.NodeSelector) ([]kusttypes.Patch, error) { + return render( + cluster.Resources, + "patches/csi-node-node-affinity.yaml", + map[string]any{ + "NODE_AFFINITY": affinity, + }) +} + func ClusterHAControllerNodeSelectorPatch(selector map[string]string) ([]kusttypes.Patch, error) { return render( cluster.Resources, @@ -64,6 +110,15 @@ func ClusterHAControllerNodeSelectorPatch(selector map[string]string) ([]kusttyp }) } +func ClusterHAControllerNodeAffinityPatch(affinity *corev1.NodeSelector) ([]kusttypes.Patch, error) { + return render( + cluster.Resources, + "patches/ha-controller-node-affinity.yaml", + map[string]any{ + "NODE_AFFINITY": affinity, + }) +} + func ClusterApiTLSPatch(apiSecretName, clientSecretName string) ([]kusttypes.Patch, error) { return render( cluster.Resources, diff --git a/internal/controller/patches_test.go b/internal/controller/patches_test.go index 5f40092d..4bac0291 100644 --- a/internal/controller/patches_test.go +++ b/internal/controller/patches_test.go @@ -5,6 +5,7 @@ import ( cmmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" kusttypes "sigs.k8s.io/kustomize/api/types" "github.com/piraeusdatastore/piraeus-operator/v2/internal/controller" @@ -37,12 +38,74 @@ func TestPatches(t *testing.T) { }) }, }, + { + name: "ClusterLinstorControllerNodeSelector", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterLinstorControllerNodeSelector(map[string]string{"foo": "bar"}) + }, + }, + { + name: "ClusterLinstorControllerNodeAffinityPatch", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterLinstorControllerNodeAffinityPatch(&corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "example.com/label", + Operator: corev1.NodeSelectorOpDoesNotExist, + }}}}, + }) + }, + }, + { + name: "ClusterCSIControllerNodeSelector", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterCSIControllerNodeSelector(map[string]string{"foo": "bar"}) + }, + }, + { + name: "ClusterCSIControllerNodeAffinityPatch", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterCSIControllerNodeAffinityPatch(&corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "example.com/label", + Operator: corev1.NodeSelectorOpDoesNotExist, + }}}}, + }) + }, + }, { name: "ClusterCSINodeSelectorPatch", call: func() ([]kusttypes.Patch, error) { return controller.ClusterCSINodeSelectorPatch(map[string]string{"foo": "bar"}) }, }, + { + name: "ClusterCSINodeNodeAffinityPatch", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterCSINodeNodeAffinityPatch(&corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "example.com/label", + Operator: corev1.NodeSelectorOpDoesNotExist, + }}}}, + }) + }, + }, + { + name: "ClusterHAControllerNodeSelectorPatch", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterHAControllerNodeSelectorPatch(map[string]string{"foo": "bar"}) + }, + }, + { + name: "ClusterHAControllerNodeAffinityPatch", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterHAControllerNodeAffinityPatch(&corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "example.com/label", + Operator: corev1.NodeSelectorOpDoesNotExist, + }}}}, + }) + }, + }, { name: "PullSecretPatch", call: func() ([]kusttypes.Patch, error) { diff --git a/pkg/merge/linstorsatelliteconfiguration.go b/pkg/merge/linstorsatelliteconfiguration.go index fac93f6d..6e510213 100644 --- a/pkg/merge/linstorsatelliteconfiguration.go +++ b/pkg/merge/linstorsatelliteconfiguration.go @@ -3,6 +3,9 @@ package merge import ( "sort" + corev1 "k8s.io/api/core/v1" + schedulingcorev1 "k8s.io/component-helpers/scheduling/corev1" + piraeusv1 "github.com/piraeusdatastore/piraeus-operator/v2/api/v1" ) @@ -12,7 +15,7 @@ import ( // * Concatenating all patches in the matching configs // * Merging all properties by name. A property defined in a "later" config overrides previous property definitions. // * Merging all storage pools by name. A storage pool defined in a "later" config overrides previous property definitions. -func SatelliteConfigurations(nodeLabels map[string]string, configs ...piraeusv1.LinstorSatelliteConfiguration) *piraeusv1.LinstorSatelliteConfiguration { +func SatelliteConfigurations(node *corev1.Node, configs ...piraeusv1.LinstorSatelliteConfiguration) *piraeusv1.LinstorSatelliteConfiguration { result := &piraeusv1.LinstorSatelliteConfiguration{} propsMap := make(map[string]*piraeusv1.LinstorNodeProperty) @@ -21,10 +24,16 @@ func SatelliteConfigurations(nodeLabels map[string]string, configs ...piraeusv1. for i := range configs { cfg := &configs[i] - if !SubsetOf(cfg.Spec.NodeSelector, nodeLabels) { + if !SubsetOf(cfg.Spec.NodeSelector, node.ObjectMeta.Labels) { continue } + if cfg.Spec.NodeAffinity != nil { + if matches, _ := schedulingcorev1.MatchNodeSelectorTerms(node, cfg.Spec.NodeAffinity); !matches { + continue + } + } + for j := range cfg.Spec.Properties { propsMap[cfg.Spec.Properties[j].Name] = &cfg.Spec.Properties[j] } diff --git a/pkg/merge/linstorsatelliteconfiguration_test.go b/pkg/merge/linstorsatelliteconfiguration_test.go index add903c9..c55b281f 100644 --- a/pkg/merge/linstorsatelliteconfiguration_test.go +++ b/pkg/merge/linstorsatelliteconfiguration_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" piraeusv1 "github.com/piraeusdatastore/piraeus-operator/v2/api/v1" "github.com/piraeusdatastore/piraeus-operator/v2/pkg/merge" @@ -71,6 +73,28 @@ var ( }}, }, } + Config4 = piraeusv1.LinstorSatelliteConfiguration{ + Spec: piraeusv1.LinstorSatelliteConfigurationSpec{ + NodeAffinity: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "config4", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"false"}, + }}, + }, + { + MatchFields: []corev1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"complex-filter-positive"}, + }}, + }, + }, + }, + }, + } ) func TestMergeSatelliteConfigurations(t *testing.T) { @@ -136,6 +160,24 @@ func TestMergeSatelliteConfigurations(t *testing.T) { }, }, }, + { + name: "complex-filter-negative", + labels: map[string]string{"config4": "false"}, + configs: []piraeusv1.LinstorSatelliteConfiguration{Config4}, + result: &piraeusv1.LinstorSatelliteConfiguration{}, + }, + { + name: "complex-filter-positive", + labels: map[string]string{"config4": "false"}, + configs: []piraeusv1.LinstorSatelliteConfiguration{Config4}, + result: &piraeusv1.LinstorSatelliteConfiguration{ + Spec: piraeusv1.LinstorSatelliteConfigurationSpec{ + Patches: Config4.Spec.Patches, + StoragePools: Config4.Spec.StoragePools, + Properties: Config4.Spec.Properties, + }, + }, + }, } for i := range testcases { @@ -143,7 +185,10 @@ func TestMergeSatelliteConfigurations(t *testing.T) { t.Run(tcase.name, func(t *testing.T) { t.Parallel() - actual := merge.SatelliteConfigurations(tcase.labels, tcase.configs...) + actual := merge.SatelliteConfigurations(&corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: tcase.name, + Labels: tcase.labels, + }}, tcase.configs...) assert.Equal(t, tcase.result, actual) }) } diff --git a/pkg/resources/cluster/patches/csi-controller-node-affinity.yaml b/pkg/resources/cluster/patches/csi-controller-node-affinity.yaml new file mode 100644 index 00000000..52be1737 --- /dev/null +++ b/pkg/resources/cluster/patches/csi-controller-node-affinity.yaml @@ -0,0 +1,17 @@ +--- +- target: + group: apps + version: v1 + kind: Deployment + name: linstor-csi-controller + patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: linstor-csi-controller + spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: $NODE_AFFINITY diff --git a/pkg/resources/cluster/patches/csi-controller-selector.yaml b/pkg/resources/cluster/patches/csi-controller-selector.yaml new file mode 100644 index 00000000..812cfb0d --- /dev/null +++ b/pkg/resources/cluster/patches/csi-controller-selector.yaml @@ -0,0 +1,15 @@ +--- +- target: + group: apps + version: v1 + kind: Deployment + name: linstor-csi-controller + patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: linstor-csi-controller + spec: + template: + spec: + nodeSelector: $NODE_SELECTOR diff --git a/pkg/resources/cluster/patches/csi-node-node-affinity.yaml b/pkg/resources/cluster/patches/csi-node-node-affinity.yaml new file mode 100644 index 00000000..1f421cff --- /dev/null +++ b/pkg/resources/cluster/patches/csi-node-node-affinity.yaml @@ -0,0 +1,17 @@ +--- +- target: + group: apps + version: v1 + kind: DaemonSet + name: linstor-csi-node + patch: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: linstor-csi-node + spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: $NODE_AFFINITY diff --git a/pkg/resources/cluster/patches/ha-controller-node-affinity.yaml b/pkg/resources/cluster/patches/ha-controller-node-affinity.yaml new file mode 100644 index 00000000..7f16845e --- /dev/null +++ b/pkg/resources/cluster/patches/ha-controller-node-affinity.yaml @@ -0,0 +1,17 @@ +--- +- target: + group: apps + version: v1 + kind: DaemonSet + name: ha-controller + patch: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: ha-controller + spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: $NODE_AFFINITY diff --git a/pkg/resources/cluster/patches/linstor-controller-node-affinity.yaml b/pkg/resources/cluster/patches/linstor-controller-node-affinity.yaml new file mode 100644 index 00000000..21c89882 --- /dev/null +++ b/pkg/resources/cluster/patches/linstor-controller-node-affinity.yaml @@ -0,0 +1,17 @@ +--- +- target: + group: apps + version: v1 + kind: Deployment + name: linstor-controller + patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: linstor-controller + spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: $NODE_AFFINITY diff --git a/pkg/resources/cluster/patches/linstor-controller-selector.yaml b/pkg/resources/cluster/patches/linstor-controller-selector.yaml new file mode 100644 index 00000000..d915c129 --- /dev/null +++ b/pkg/resources/cluster/patches/linstor-controller-selector.yaml @@ -0,0 +1,15 @@ +--- +- target: + group: apps + version: v1 + kind: Deployment + name: linstor-controller + patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: linstor-controller + spec: + template: + spec: + nodeSelector: $NODE_SELECTOR