From 0f3b0bcf71708c16aab826acd625ce8c3af8c75b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20=22WanzenBug=22=20Wanzenb=C3=B6ck?= Date: Thu, 31 Aug 2023 09:02:28 +0200 Subject: [PATCH] Support complex node affinity in cluster spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the nodeAffinity, like one would set it on pods to allow more complex deployment scenarios. We need to respect these settings when creating Satellites, but also when creating our own deployments. For consistency, the same options are also available on sht satellite configuration resources. Signed-off-by: Moritz "WanzenBug" Wanzenböck --- .github/workflows/checks.yml | 4 +- CHANGELOG.md | 1 + Dockerfile | 2 +- api/v1/linstorcluster_types.go | 6 + api/v1/linstorsatelliteconfiguration_types.go | 6 + api/v1/zz_generated.deepcopy.go | 11 ++ charts/piraeus/templates/crds.yaml | 170 ++++++++++++++++++ .../crd/bases/piraeus.io_linstorclusters.yaml | 85 +++++++++ ...eus.io_linstorsatelliteconfigurations.yaml | 85 +++++++++ docs/reference/linstorcluster.md | 36 ++++ .../linstorsatelliteconfiguration.md | 27 +++ go.mod | 9 +- go.sum | 14 +- .../controller/linstorcluster_controller.go | 57 +++++- internal/controller/linstorcluster_test.go | 52 +++++- internal/controller/patches.go | 55 ++++++ internal/controller/patches_test.go | 63 +++++++ pkg/merge/linstorsatelliteconfiguration.go | 13 +- .../linstorsatelliteconfiguration_test.go | 47 ++++- .../patches/csi-controller-node-affinity.yaml | 17 ++ .../patches/csi-controller-selector.yaml | 15 ++ .../patches/csi-node-node-affinity.yaml | 17 ++ .../patches/ha-controller-node-affinity.yaml | 17 ++ .../linstor-controller-node-affinity.yaml | 17 ++ .../patches/linstor-controller-selector.yaml | 15 ++ 25 files changed, 822 insertions(+), 19 deletions(-) create mode 100644 pkg/resources/cluster/patches/csi-controller-node-affinity.yaml create mode 100644 pkg/resources/cluster/patches/csi-controller-selector.yaml create mode 100644 pkg/resources/cluster/patches/csi-node-node-affinity.yaml create mode 100644 pkg/resources/cluster/patches/ha-controller-node-affinity.yaml create mode 100644 pkg/resources/cluster/patches/linstor-controller-node-affinity.yaml create mode 100644 pkg/resources/cluster/patches/linstor-controller-selector.yaml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e759b12d..8eebcfa7 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - uses: golangci/golangci-lint-action@v3 with: args: --timeout=3m @@ -29,7 +29,7 @@ jobs: - name: Setup Go environment uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - uses: actions/setup-python@v4 - name: Run pre-commit checks on changes files uses: pre-commit/action@v3.0.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index ccc9e2c4..cfbabe4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add image configuration for CSI sidecars. - Check kernel module parameters for DRBD on load. - Automatically set SELinux labels when loading kernel modules. +- Allow more complex node selection by adding `LinstorCluster.spec.nodeAffinity`. ### Changed diff --git a/Dockerfile b/Dockerfile index 45e406d5..f1dbc43a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.4 # Build the manager binary -FROM --platform=$BUILDPLATFORM golang:1.20 as builder +FROM --platform=$BUILDPLATFORM golang:1.21 as builder WORKDIR /workspace # Copy the Go Modules manifests diff --git a/api/v1/linstorcluster_types.go b/api/v1/linstorcluster_types.go index 891bf5ce..13589853 100644 --- a/api/v1/linstorcluster_types.go +++ b/api/v1/linstorcluster_types.go @@ -18,6 +18,7 @@ package v1 import ( cmmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -38,6 +39,11 @@ type LinstorClusterSpec struct { // +kubebuilder:validation:Optional NodeSelector map[string]string `json:"nodeSelector,omitempty"` + // NodeAffinity selects the nodes on which LINSTOR Satellite will be deployed. + // See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + // +kubebuilder:validation:Optional + NodeAffinity *corev1.NodeSelector `json:"nodeAffinity,omitempty"` + // Properties to apply on the cluster level. // // Use to create default settings for DRBD that should apply to all resources or to configure some other cluster diff --git a/api/v1/linstorsatelliteconfiguration_types.go b/api/v1/linstorsatelliteconfiguration_types.go index af4292ee..2ee0009f 100644 --- a/api/v1/linstorsatelliteconfiguration_types.go +++ b/api/v1/linstorsatelliteconfiguration_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -30,6 +31,11 @@ type LinstorSatelliteConfigurationSpec struct { // +kubebuilder:validation:Optional NodeSelector map[string]string `json:"nodeSelector,omitempty"` + // NodeAffinity selects which LinstorSatellite resources this spec should be applied to. + // See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + // +kubebuilder:validation:Optional + NodeAffinity *corev1.NodeSelector `json:"nodeAffinity,omitempty"` + // Patches is a list of kustomize patches to apply. // // See https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/ for how to create patches. diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index eef02734..430b0701 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -22,6 +22,7 @@ package v1 import ( metav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" apismetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -140,6 +141,11 @@ func (in *LinstorClusterSpec) DeepCopyInto(out *LinstorClusterSpec) { (*out)[key] = val } } + if in.NodeAffinity != nil { + in, out := &in.NodeAffinity, &out.NodeAffinity + *out = new(corev1.NodeSelector) + (*in).DeepCopyInto(*out) + } if in.Properties != nil { in, out := &in.Properties, &out.Properties *out = make([]LinstorControllerProperty, len(*in)) @@ -485,6 +491,11 @@ func (in *LinstorSatelliteConfigurationSpec) DeepCopyInto(out *LinstorSatelliteC (*out)[key] = val } } + if in.NodeAffinity != nil { + in, out := &in.NodeAffinity, &out.NodeAffinity + *out = new(corev1.NodeSelector) + (*in).DeepCopyInto(*out) + } if in.Patches != nil { in, out := &in.Patches, &out.Patches *out = make([]Patch, len(*in)) diff --git a/charts/piraeus/templates/crds.yaml b/charts/piraeus/templates/crds.yaml index 9f11c78c..b343e941 100644 --- a/charts/piraeus/templates/crds.yaml +++ b/charts/piraeus/templates/crds.yaml @@ -129,6 +129,91 @@ spec: for accessing remotes for backups. See https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/#s-encrypt_commands for more information." type: string + nodeAffinity: + description: NodeAffinity selects the nodes on which LINSTOR Satellite + will be deployed. See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms + are ORed. + items: + description: A null or empty node selector term matches no objects. + The requirements of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's + labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's + fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic nodeSelector: additionalProperties: type: string @@ -569,6 +654,91 @@ spec: connections on behalf of DRBD." type: boolean type: object + nodeAffinity: + description: NodeAffinity selects which LinstorSatellite resources + this spec should be applied to. See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms + are ORed. + items: + description: A null or empty node selector term matches no objects. + The requirements of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's + labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's + fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic nodeSelector: additionalProperties: type: string diff --git a/config/crd/bases/piraeus.io_linstorclusters.yaml b/config/crd/bases/piraeus.io_linstorclusters.yaml index 047c6276..267961bc 100644 --- a/config/crd/bases/piraeus.io_linstorclusters.yaml +++ b/config/crd/bases/piraeus.io_linstorclusters.yaml @@ -128,6 +128,91 @@ spec: for accessing remotes for backups. See https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/#s-encrypt_commands for more information." type: string + nodeAffinity: + description: NodeAffinity selects the nodes on which LINSTOR Satellite + will be deployed. See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms + are ORed. + items: + description: A null or empty node selector term matches no objects. + The requirements of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's + labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's + fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic nodeSelector: additionalProperties: type: string diff --git a/config/crd/bases/piraeus.io_linstorsatelliteconfigurations.yaml b/config/crd/bases/piraeus.io_linstorsatelliteconfigurations.yaml index 0ae34f3a..3b634e31 100644 --- a/config/crd/bases/piraeus.io_linstorsatelliteconfigurations.yaml +++ b/config/crd/bases/piraeus.io_linstorsatelliteconfigurations.yaml @@ -73,6 +73,91 @@ spec: connections on behalf of DRBD." type: boolean type: object + nodeAffinity: + description: NodeAffinity selects which LinstorSatellite resources + this spec should be applied to. See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms + are ORed. + items: + description: A null or empty node selector term matches no objects. + The requirements of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's + labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's + fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists, + DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator + is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. If the operator is Gt or Lt, + the values array must have a single element, which + will be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic nodeSelector: additionalProperties: type: string diff --git a/docs/reference/linstorcluster.md b/docs/reference/linstorcluster.md index 70fb8c82..d4de4dc4 100644 --- a/docs/reference/linstorcluster.md +++ b/docs/reference/linstorcluster.md @@ -18,6 +18,9 @@ excluded by the selector will not be able to run any workload using a Piraeus vo If empty (the default), Piraeus will be deployed on all nodes in the cluster. +When this is used together with `.spec.nodeAffinity`, both need to match in order for +a node to run Piraeus. + #### Example This example restricts Piraeus Datastore to nodes matching `example.com/storage: "yes"`: @@ -32,6 +35,39 @@ spec: example.com/storage: "yes" ``` +### `.spec.nodeAffinity` + +Selects on which nodes Piraeus Datastore should be deployed. Nodes that are +excluded by the affinity will not be able to run any workload using a Piraeus volume. + +If empty (the default), Piraeus will be deployed on all nodes in the cluster. + +When this is used together with `.spec.nodeSelector`, both need to match in order for +a node to run Piraeus. + +#### Example + +This example restricts Piraeus Datastore to nodes in zones `a` and `b`, but not on `control-plane` nodes: + +```yaml +apiVersion: piraeus.io/v1 +kind: LinstorCluster +metadata: + name: linstorcluster +spec: + nodeAffinity: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - a + - b + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist +``` + + ### `.spec.repository` Sets the default image registry to use for all Piraeus images. The full image name is diff --git a/docs/reference/linstorsatelliteconfiguration.md b/docs/reference/linstorsatelliteconfiguration.md index e8a67d5a..d95cd150 100644 --- a/docs/reference/linstorsatelliteconfiguration.md +++ b/docs/reference/linstorsatelliteconfiguration.md @@ -27,6 +27,33 @@ spec: value: "no" ``` +### `.spec.nodeAffinity` + +Selects which nodes the LinstorSatelliteConfiguration should apply to. If empty, the configuration applies to all nodes. + +When this is used together with `.spec.nodeAffinity`, both need to match in order for the configuration to apply to a +node. + +#### Example + +This example sets the `AutoplaceTarget` property to `no` on all non-worker nodes: + +```yaml +apiVersion: piraeus.io/v1 +kind: LinstorSatelliteConfiguration +metadata: + name: disabled-nodes +spec: + nodeAffinity: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + properties: + - name: AutoplaceTarget + value: "no" +``` + ### `.spec.properties` Sets the given properties on the LINSTOR Satellite level. diff --git a/go.mod b/go.mod index 27c86fa9..1803e382 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/piraeusdatastore/piraeus-operator/v2 -go 1.20 +go 1.21 require ( github.com/LINBIT/golinstor v0.49.0 @@ -14,9 +14,10 @@ require ( golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/time v0.4.0 gonum.org/v1/gonum v0.14.0 - k8s.io/api v0.28.3 - k8s.io/apimachinery v0.28.3 - k8s.io/client-go v0.28.3 + k8s.io/api v0.28.4 + k8s.io/apimachinery v0.28.4 + k8s.io/client-go v0.28.4 + k8s.io/component-helpers v0.28.4 sigs.k8s.io/controller-runtime v0.16.3 sigs.k8s.io/kustomize/api v0.15.0 sigs.k8s.io/kustomize/kyaml v0.15.0 diff --git a/go.sum b/go.sum index bccaf99d..fa9e2576 100644 --- a/go.sum +++ b/go.sum @@ -245,16 +245,18 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= -k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= +k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY= +k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0= k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc= -k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= -k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= -k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= -k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= +k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8= +k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= +k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY= +k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4= k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= +k8s.io/component-helpers v0.28.4 h1:+X9VXT5+jUsRdC26JyMZ8Fjfln7mSjgumafocE509C4= +k8s.io/component-helpers v0.28.4/go.mod h1:8LzMalOQ0K10tkBJWBWq8h0HTI9HDPx4WT3QvTFn9Ro= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= diff --git a/internal/controller/linstorcluster_controller.go b/internal/controller/linstorcluster_controller.go index a138a610..32f7e614 100644 --- a/internal/controller/linstorcluster_controller.go +++ b/internal/controller/linstorcluster_controller.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + "slices" "sort" "strings" "time" @@ -36,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + schedulingcorev1 "k8s.io/component-helpers/scheduling/corev1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -147,6 +149,13 @@ func (r *LinstorClusterReconciler) reconcileAppliedResource(ctx context.Context, return err } + if lcluster.Spec.NodeAffinity != nil { + satelliteNodes.Items = slices.DeleteFunc(satelliteNodes.Items, func(node corev1.Node) bool { + matches, _ := schedulingcorev1.MatchNodeSelectorTerms(&node, lcluster.Spec.NodeAffinity) + return !matches + }) + } + satelliteConfigs := piraeusiov1.LinstorSatelliteConfigurationList{} err = r.Client.List(ctx, &satelliteConfigs) if err != nil { @@ -299,9 +308,22 @@ func (r *LinstorClusterReconciler) kustomizeControllerResources(lcluster *piraeu return resmap.New(), nil } - var patches []kusttypes.Patch resourceDirs := []string{"controller"} + patches, err := ClusterLinstorControllerNodeSelector(lcluster.Spec.NodeSelector) + if err != nil { + return nil, err + } + + if lcluster.Spec.NodeAffinity != nil { + p, err := ClusterLinstorControllerNodeAffinityPatch(lcluster.Spec.NodeAffinity) + if err != nil { + return nil, err + } + + patches = append(patches, p...) + } + if lcluster.Spec.LinstorPassphraseSecret != "" { p, err := ClusterLinstorPassphrasePatch(lcluster.Spec.LinstorPassphraseSecret) if err != nil { @@ -390,6 +412,13 @@ func (r *LinstorClusterReconciler) kustomizeCsiResources(lcluster *piraeusiov1.L return nil, err } + p, err := ClusterCSIControllerNodeSelector(lcluster.Spec.NodeSelector) + if err != nil { + return nil, err + } + + patches = append(patches, p...) + endpointPatches, err := ClusterApiEndpointPatch(LinstorControllerUrl(lcluster)) if err != nil { return nil, err @@ -397,6 +426,21 @@ func (r *LinstorClusterReconciler) kustomizeCsiResources(lcluster *piraeusiov1.L patches = append(patches, endpointPatches...) + if lcluster.Spec.NodeAffinity != nil { + nodePatches, err := ClusterCSINodeNodeAffinityPatch(lcluster.Spec.NodeAffinity) + if err != nil { + return nil, err + } + + controllerPatches, err := ClusterCSIControllerNodeAffinityPatch(lcluster.Spec.NodeAffinity) + if err != nil { + return nil, err + } + + patches = append(patches, nodePatches...) + patches = append(patches, controllerPatches...) + } + if lcluster.Spec.ApiTLS != nil { controllerSecret := lcluster.Spec.ApiTLS.GetCsiControllerSecretName() nodeSecret := lcluster.Spec.ApiTLS.GetCsiNodeSecretName() @@ -444,6 +488,15 @@ func (r *LinstorClusterReconciler) kustomizeHAControllerResources(lcluster *pira return nil, err } + if lcluster.Spec.NodeAffinity != nil { + p, err := ClusterHAControllerNodeAffinityPatch(lcluster.Spec.NodeAffinity) + if err != nil { + return nil, err + } + + patches = append(patches, p...) + } + return r.kustomize([]string{"ha-controller"}, lcluster, imgs, patches...) } @@ -499,7 +552,7 @@ func (r *LinstorClusterReconciler) kustomizeLinstorSatellite(lcluster *piraeusio patches := []utils.JsonPatch{renamePatch, repositoryPatch, clusterRefPatch} - cfg := merge.SatelliteConfigurations(node.ObjectMeta.Labels, configs...) + cfg := merge.SatelliteConfigurations(node, configs...) if cfg.Spec.InternalTLS != nil { patches = append(patches, utils.JsonPatch{ diff --git a/internal/controller/linstorcluster_test.go b/internal/controller/linstorcluster_test.go index 01e95ba0..59970777 100644 --- a/internal/controller/linstorcluster_test.go +++ b/internal/controller/linstorcluster_test.go @@ -59,7 +59,10 @@ var _ = Describe("LinstorCluster controller", func() { Describe("with cluster nodes present", func() { BeforeEach(func(ctx context.Context) { err := k8sClient.Create(ctx, &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{Name: "node-1a", Labels: map[string]string{"topology.kubernetes.io/zone": "a"}}, + ObjectMeta: metav1.ObjectMeta{Name: "node-1a", Labels: map[string]string{ + "topology.kubernetes.io/zone": "a", + "example.com/exclude": "yes", + }}, }) Expect(err).NotTo(HaveOccurred()) @@ -241,6 +244,53 @@ var _ = Describe("LinstorCluster controller", func() { return controller.Image }, DefaultTimeout, DefaultCheckInterval).Should(HavePrefix("piraeus.io/test")) }) + + It("should apply affinity set on the cluster resource", func(ctx context.Context) { + Eventually(func() bool { + var satellites piraeusiov1.LinstorSatelliteList + err := k8sClient.List(ctx, &satellites) + Expect(err).NotTo(HaveOccurred()) + + return len(satellites.Items) == 3 + }, DefaultTimeout, DefaultCheckInterval).Should(BeTrue()) + + var cluster piraeusiov1.LinstorCluster + err := k8sClient.Get(ctx, types.NamespacedName{Name: "default"}, &cluster) + Expect(err).NotTo(HaveOccurred()) + + cluster.Spec.NodeAffinity = &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "topology.kubernetes.io/zone", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"b"}, + }, + { + Key: "example.com/exclude", + Operator: corev1.NodeSelectorOpDoesNotExist, + }, + }, + }}, + } + + err = k8sClient.Update(ctx, &cluster) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []string { + var satellites piraeusiov1.LinstorSatelliteList + err := k8sClient.List(ctx, &satellites) + Expect(err).NotTo(HaveOccurred()) + + var result []string + for i := range satellites.Items { + if satellites.Items[i].DeletionTimestamp == nil { + result = append(result, satellites.Items[i].Name) + } + } + return result + }, DefaultTimeout, DefaultCheckInterval).Should(ConsistOf("node-2a")) + }) }) }) diff --git a/internal/controller/patches.go b/internal/controller/patches.go index 5f58ea6d..1ce8e104 100644 --- a/internal/controller/patches.go +++ b/internal/controller/patches.go @@ -7,6 +7,7 @@ import ( "strings" cmmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" kusttypes "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/yaml" @@ -46,6 +47,42 @@ func ClusterLinstorInternalTLSCertManagerPatch(secretName string, issuer *cmmeta ) } +func ClusterLinstorControllerNodeSelector(selector map[string]string) ([]kusttypes.Patch, error) { + return render( + cluster.Resources, + "patches/linstor-controller-selector.yaml", + map[string]any{ + "NODE_SELECTOR": selector, + }) +} + +func ClusterLinstorControllerNodeAffinityPatch(affinity *corev1.NodeSelector) ([]kusttypes.Patch, error) { + return render( + cluster.Resources, + "patches/linstor-controller-node-affinity.yaml", + map[string]any{ + "NODE_AFFINITY": affinity, + }) +} + +func ClusterCSIControllerNodeSelector(selector map[string]string) ([]kusttypes.Patch, error) { + return render( + cluster.Resources, + "patches/csi-controller-selector.yaml", + map[string]any{ + "NODE_SELECTOR": selector, + }) +} + +func ClusterCSIControllerNodeAffinityPatch(affinity *corev1.NodeSelector) ([]kusttypes.Patch, error) { + return render( + cluster.Resources, + "patches/csi-controller-node-affinity.yaml", + map[string]any{ + "NODE_AFFINITY": affinity, + }) +} + func ClusterCSINodeSelectorPatch(selector map[string]string) ([]kusttypes.Patch, error) { return render( cluster.Resources, @@ -55,6 +92,15 @@ func ClusterCSINodeSelectorPatch(selector map[string]string) ([]kusttypes.Patch, }) } +func ClusterCSINodeNodeAffinityPatch(affinity *corev1.NodeSelector) ([]kusttypes.Patch, error) { + return render( + cluster.Resources, + "patches/csi-node-node-affinity.yaml", + map[string]any{ + "NODE_AFFINITY": affinity, + }) +} + func ClusterHAControllerNodeSelectorPatch(selector map[string]string) ([]kusttypes.Patch, error) { return render( cluster.Resources, @@ -64,6 +110,15 @@ func ClusterHAControllerNodeSelectorPatch(selector map[string]string) ([]kusttyp }) } +func ClusterHAControllerNodeAffinityPatch(affinity *corev1.NodeSelector) ([]kusttypes.Patch, error) { + return render( + cluster.Resources, + "patches/ha-controller-node-affinity.yaml", + map[string]any{ + "NODE_AFFINITY": affinity, + }) +} + func ClusterApiTLSPatch(apiSecretName, clientSecretName string) ([]kusttypes.Patch, error) { return render( cluster.Resources, diff --git a/internal/controller/patches_test.go b/internal/controller/patches_test.go index 5f40092d..4bac0291 100644 --- a/internal/controller/patches_test.go +++ b/internal/controller/patches_test.go @@ -5,6 +5,7 @@ import ( cmmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" kusttypes "sigs.k8s.io/kustomize/api/types" "github.com/piraeusdatastore/piraeus-operator/v2/internal/controller" @@ -37,12 +38,74 @@ func TestPatches(t *testing.T) { }) }, }, + { + name: "ClusterLinstorControllerNodeSelector", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterLinstorControllerNodeSelector(map[string]string{"foo": "bar"}) + }, + }, + { + name: "ClusterLinstorControllerNodeAffinityPatch", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterLinstorControllerNodeAffinityPatch(&corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "example.com/label", + Operator: corev1.NodeSelectorOpDoesNotExist, + }}}}, + }) + }, + }, + { + name: "ClusterCSIControllerNodeSelector", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterCSIControllerNodeSelector(map[string]string{"foo": "bar"}) + }, + }, + { + name: "ClusterCSIControllerNodeAffinityPatch", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterCSIControllerNodeAffinityPatch(&corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "example.com/label", + Operator: corev1.NodeSelectorOpDoesNotExist, + }}}}, + }) + }, + }, { name: "ClusterCSINodeSelectorPatch", call: func() ([]kusttypes.Patch, error) { return controller.ClusterCSINodeSelectorPatch(map[string]string{"foo": "bar"}) }, }, + { + name: "ClusterCSINodeNodeAffinityPatch", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterCSINodeNodeAffinityPatch(&corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "example.com/label", + Operator: corev1.NodeSelectorOpDoesNotExist, + }}}}, + }) + }, + }, + { + name: "ClusterHAControllerNodeSelectorPatch", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterHAControllerNodeSelectorPatch(map[string]string{"foo": "bar"}) + }, + }, + { + name: "ClusterHAControllerNodeAffinityPatch", + call: func() ([]kusttypes.Patch, error) { + return controller.ClusterHAControllerNodeAffinityPatch(&corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "example.com/label", + Operator: corev1.NodeSelectorOpDoesNotExist, + }}}}, + }) + }, + }, { name: "PullSecretPatch", call: func() ([]kusttypes.Patch, error) { diff --git a/pkg/merge/linstorsatelliteconfiguration.go b/pkg/merge/linstorsatelliteconfiguration.go index fac93f6d..6e510213 100644 --- a/pkg/merge/linstorsatelliteconfiguration.go +++ b/pkg/merge/linstorsatelliteconfiguration.go @@ -3,6 +3,9 @@ package merge import ( "sort" + corev1 "k8s.io/api/core/v1" + schedulingcorev1 "k8s.io/component-helpers/scheduling/corev1" + piraeusv1 "github.com/piraeusdatastore/piraeus-operator/v2/api/v1" ) @@ -12,7 +15,7 @@ import ( // * Concatenating all patches in the matching configs // * Merging all properties by name. A property defined in a "later" config overrides previous property definitions. // * Merging all storage pools by name. A storage pool defined in a "later" config overrides previous property definitions. -func SatelliteConfigurations(nodeLabels map[string]string, configs ...piraeusv1.LinstorSatelliteConfiguration) *piraeusv1.LinstorSatelliteConfiguration { +func SatelliteConfigurations(node *corev1.Node, configs ...piraeusv1.LinstorSatelliteConfiguration) *piraeusv1.LinstorSatelliteConfiguration { result := &piraeusv1.LinstorSatelliteConfiguration{} propsMap := make(map[string]*piraeusv1.LinstorNodeProperty) @@ -21,10 +24,16 @@ func SatelliteConfigurations(nodeLabels map[string]string, configs ...piraeusv1. for i := range configs { cfg := &configs[i] - if !SubsetOf(cfg.Spec.NodeSelector, nodeLabels) { + if !SubsetOf(cfg.Spec.NodeSelector, node.ObjectMeta.Labels) { continue } + if cfg.Spec.NodeAffinity != nil { + if matches, _ := schedulingcorev1.MatchNodeSelectorTerms(node, cfg.Spec.NodeAffinity); !matches { + continue + } + } + for j := range cfg.Spec.Properties { propsMap[cfg.Spec.Properties[j].Name] = &cfg.Spec.Properties[j] } diff --git a/pkg/merge/linstorsatelliteconfiguration_test.go b/pkg/merge/linstorsatelliteconfiguration_test.go index add903c9..c55b281f 100644 --- a/pkg/merge/linstorsatelliteconfiguration_test.go +++ b/pkg/merge/linstorsatelliteconfiguration_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" piraeusv1 "github.com/piraeusdatastore/piraeus-operator/v2/api/v1" "github.com/piraeusdatastore/piraeus-operator/v2/pkg/merge" @@ -71,6 +73,28 @@ var ( }}, }, } + Config4 = piraeusv1.LinstorSatelliteConfiguration{ + Spec: piraeusv1.LinstorSatelliteConfigurationSpec{ + NodeAffinity: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "config4", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"false"}, + }}, + }, + { + MatchFields: []corev1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"complex-filter-positive"}, + }}, + }, + }, + }, + }, + } ) func TestMergeSatelliteConfigurations(t *testing.T) { @@ -136,6 +160,24 @@ func TestMergeSatelliteConfigurations(t *testing.T) { }, }, }, + { + name: "complex-filter-negative", + labels: map[string]string{"config4": "false"}, + configs: []piraeusv1.LinstorSatelliteConfiguration{Config4}, + result: &piraeusv1.LinstorSatelliteConfiguration{}, + }, + { + name: "complex-filter-positive", + labels: map[string]string{"config4": "false"}, + configs: []piraeusv1.LinstorSatelliteConfiguration{Config4}, + result: &piraeusv1.LinstorSatelliteConfiguration{ + Spec: piraeusv1.LinstorSatelliteConfigurationSpec{ + Patches: Config4.Spec.Patches, + StoragePools: Config4.Spec.StoragePools, + Properties: Config4.Spec.Properties, + }, + }, + }, } for i := range testcases { @@ -143,7 +185,10 @@ func TestMergeSatelliteConfigurations(t *testing.T) { t.Run(tcase.name, func(t *testing.T) { t.Parallel() - actual := merge.SatelliteConfigurations(tcase.labels, tcase.configs...) + actual := merge.SatelliteConfigurations(&corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: tcase.name, + Labels: tcase.labels, + }}, tcase.configs...) assert.Equal(t, tcase.result, actual) }) } diff --git a/pkg/resources/cluster/patches/csi-controller-node-affinity.yaml b/pkg/resources/cluster/patches/csi-controller-node-affinity.yaml new file mode 100644 index 00000000..52be1737 --- /dev/null +++ b/pkg/resources/cluster/patches/csi-controller-node-affinity.yaml @@ -0,0 +1,17 @@ +--- +- target: + group: apps + version: v1 + kind: Deployment + name: linstor-csi-controller + patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: linstor-csi-controller + spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: $NODE_AFFINITY diff --git a/pkg/resources/cluster/patches/csi-controller-selector.yaml b/pkg/resources/cluster/patches/csi-controller-selector.yaml new file mode 100644 index 00000000..812cfb0d --- /dev/null +++ b/pkg/resources/cluster/patches/csi-controller-selector.yaml @@ -0,0 +1,15 @@ +--- +- target: + group: apps + version: v1 + kind: Deployment + name: linstor-csi-controller + patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: linstor-csi-controller + spec: + template: + spec: + nodeSelector: $NODE_SELECTOR diff --git a/pkg/resources/cluster/patches/csi-node-node-affinity.yaml b/pkg/resources/cluster/patches/csi-node-node-affinity.yaml new file mode 100644 index 00000000..1f421cff --- /dev/null +++ b/pkg/resources/cluster/patches/csi-node-node-affinity.yaml @@ -0,0 +1,17 @@ +--- +- target: + group: apps + version: v1 + kind: DaemonSet + name: linstor-csi-node + patch: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: linstor-csi-node + spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: $NODE_AFFINITY diff --git a/pkg/resources/cluster/patches/ha-controller-node-affinity.yaml b/pkg/resources/cluster/patches/ha-controller-node-affinity.yaml new file mode 100644 index 00000000..7f16845e --- /dev/null +++ b/pkg/resources/cluster/patches/ha-controller-node-affinity.yaml @@ -0,0 +1,17 @@ +--- +- target: + group: apps + version: v1 + kind: DaemonSet + name: ha-controller + patch: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: ha-controller + spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: $NODE_AFFINITY diff --git a/pkg/resources/cluster/patches/linstor-controller-node-affinity.yaml b/pkg/resources/cluster/patches/linstor-controller-node-affinity.yaml new file mode 100644 index 00000000..21c89882 --- /dev/null +++ b/pkg/resources/cluster/patches/linstor-controller-node-affinity.yaml @@ -0,0 +1,17 @@ +--- +- target: + group: apps + version: v1 + kind: Deployment + name: linstor-controller + patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: linstor-controller + spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: $NODE_AFFINITY diff --git a/pkg/resources/cluster/patches/linstor-controller-selector.yaml b/pkg/resources/cluster/patches/linstor-controller-selector.yaml new file mode 100644 index 00000000..d915c129 --- /dev/null +++ b/pkg/resources/cluster/patches/linstor-controller-selector.yaml @@ -0,0 +1,15 @@ +--- +- target: + group: apps + version: v1 + kind: Deployment + name: linstor-controller + patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: linstor-controller + spec: + template: + spec: + nodeSelector: $NODE_SELECTOR