From 026f773cbcd337a560e504a077c181c653c7ce32 Mon Sep 17 00:00:00 2001 From: mamutmk5 <3045922+mamutmk5@users.noreply.github.com> Date: Mon, 4 Dec 2023 16:20:29 +0100 Subject: [PATCH] BC-5423 - Add POD affinity and anti-affinity (#723) To Improve the distribution of PODs to the least number of kubernetes worker nodes who is Required but also to secure that the PODs of one Deployment are distributed over the Fault Zones and Node pools the pod affinity and anti affinity are set. Affernety number calculation: Assign Number for an host = 0 - 20 X Fault Zone - 10 X Node Pool + 9 X Host Host = 1 if on the host runs an POD from the Schulcloud-Verbund on any namespace Host = 0 if on the host runs no POD from the Schulcloud-Verbund on any namespace Node Pool = 1 if on a host of this node pool runs an POD from the Deployment Node Pool = 0 if on no host of this node pool runs an POD from the Deployment Fault Pool = 1 if on a host of this fault zone runs an POD from the Deployment Fault Pool = 0 if on no host of this fault zone uns an POD from the Deployment --- ansible/group_vars/all/affinity.yml | 3 ++ .../roles/clamav/templates/deployment.yml.j2 | 37 +++++++++++++++++++ .../dof_etherpad/templates/deployment.yml.j2 | 37 +++++++++++++++++++ .../templates/deployment.yml.j2 | 37 +++++++++++++++++++ .../templates/deployment.yml.j2 | 15 ++++++++ .../dof_mongo/templates/deployment.yml.j2 | 15 ++++++++ .../templates/deployment.yml.j2 | 15 ++++++++ .../dof_redis/templates/deployment.yml.j2 | 37 +++++++++++++++++++ .../erwin-idm/templates/deployment.yml.j2 | 37 +++++++++++++++++++ .../roles/hydra/templates/deployment.yml.j2 | 37 +++++++++++++++++++ .../maildrop/templates/deployment.yml.j2 | 15 ++++++++ .../oidcmock/templates/deployment.yml.j2 | 15 ++++++++ .../rocketchat/templates/deployment.yml.j2 | 37 +++++++++++++++++++ .../rocketchat/templates/fixup-job.yml.j2 | 15 ++++++++ .../roles/storage/templates/deployment.yml.j2 | 37 +++++++++++++++++++ 15 files changed, 389 insertions(+) create mode 100644 ansible/group_vars/all/affinity.yml diff --git a/ansible/group_vars/all/affinity.yml b/ansible/group_vars/all/affinity.yml new file mode 100644 index 000000000..9cb5c16e1 --- /dev/null +++ b/ansible/group_vars/all/affinity.yml @@ -0,0 +1,3 @@ +AFFINITY_ENABLE: true +ANIT_AFFINITY_NODEPOOL_ENABLE: true +ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY: "cloud.ionos.com/nodepool-name" diff --git a/ansible/roles/clamav/templates/deployment.yml.j2 b/ansible/roles/clamav/templates/deployment.yml.j2 index d3ceefdf1..792f82e1e 100644 --- a/ansible/roles/clamav/templates/deployment.yml.j2 +++ b/ansible/roles/clamav/templates/deployment.yml.j2 @@ -46,3 +46,40 @@ spec: requests: cpu: {{ CLAMAV_CPU_MIN|default("100m", true) }} memory: {{ CLAMAV_MEMORY_MIN|default("1Gi", true) }} +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 9 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: +{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %} + - weight: 10 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - clamav + topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }} +{% endif %} + - weight: 20 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - clamav + topologyKey: "topology.kubernetes.io/zone" +{% endif %} diff --git a/ansible/roles/dof_etherpad/templates/deployment.yml.j2 b/ansible/roles/dof_etherpad/templates/deployment.yml.j2 index b43839709..5157482b3 100644 --- a/ansible/roles/dof_etherpad/templates/deployment.yml.j2 +++ b/ansible/roles/dof_etherpad/templates/deployment.yml.j2 @@ -74,3 +74,40 @@ spec: items: - key: ETHERPAD_API_KEY path: APIKEY.txt +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 9 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: +{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %} + - weight: 10 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - etherpad + topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }} +{% endif %} + - weight: 20 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - etherpad + topologyKey: "topology.kubernetes.io/zone" +{% endif %} diff --git a/ansible/roles/dof_etherpad_nginx/templates/deployment.yml.j2 b/ansible/roles/dof_etherpad_nginx/templates/deployment.yml.j2 index 3f2745a27..567b14a8e 100644 --- a/ansible/roles/dof_etherpad_nginx/templates/deployment.yml.j2 +++ b/ansible/roles/dof_etherpad_nginx/templates/deployment.yml.j2 @@ -77,3 +77,40 @@ spec: items: - key: default.conf path: default.conf +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 9 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: +{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %} + - weight: 10 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - etherpad-nginx + topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }} +{% endif %} + - weight: 20 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - etherpad-nginx + topologyKey: "topology.kubernetes.io/zone" +{% endif %} diff --git a/ansible/roles/dof_mailcatcher/templates/deployment.yml.j2 b/ansible/roles/dof_mailcatcher/templates/deployment.yml.j2 index fc5fb5b96..15042e5d0 100644 --- a/ansible/roles/dof_mailcatcher/templates/deployment.yml.j2 +++ b/ansible/roles/dof_mailcatcher/templates/deployment.yml.j2 @@ -53,3 +53,18 @@ spec: requests: cpu: {{ MAILCATCHER_CPU_REQUESTS|default("100m", true) }} memory: {{ MAILCATCHER_MEMORY_REQUESTS|default("256Mi", true) }} +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} +{% endif %} diff --git a/ansible/roles/dof_mongo/templates/deployment.yml.j2 b/ansible/roles/dof_mongo/templates/deployment.yml.j2 index 1674ce287..537b93b99 100644 --- a/ansible/roles/dof_mongo/templates/deployment.yml.j2 +++ b/ansible/roles/dof_mongo/templates/deployment.yml.j2 @@ -83,3 +83,18 @@ spec: - name: mongodb-data-pv persistentVolumeClaim: claimName: mongo-pvc +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} +{% endif %} diff --git a/ansible/roles/dof_postgresql/templates/deployment.yml.j2 b/ansible/roles/dof_postgresql/templates/deployment.yml.j2 index e4fd0d78c..26250db7e 100644 --- a/ansible/roles/dof_postgresql/templates/deployment.yml.j2 +++ b/ansible/roles/dof_postgresql/templates/deployment.yml.j2 @@ -67,3 +67,18 @@ spec: items: - key: 01_erwinidm.sql path: 01_erwinidm.sql +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} +{% endif %} diff --git a/ansible/roles/dof_redis/templates/deployment.yml.j2 b/ansible/roles/dof_redis/templates/deployment.yml.j2 index 291cedd76..ef3bfc0eb 100644 --- a/ansible/roles/dof_redis/templates/deployment.yml.j2 +++ b/ansible/roles/dof_redis/templates/deployment.yml.j2 @@ -58,3 +58,40 @@ spec: - name: redis-data-pv persistentVolumeClaim: claimName: redis-pvc +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 9 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: +{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %} + - weight: 10 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - redis + topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }} +{% endif %} + - weight: 20 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - redis + topologyKey: "topology.kubernetes.io/zone" +{% endif %} diff --git a/ansible/roles/erwin-idm/templates/deployment.yml.j2 b/ansible/roles/erwin-idm/templates/deployment.yml.j2 index f9ee7600f..c6c26fa70 100644 --- a/ansible/roles/erwin-idm/templates/deployment.yml.j2 +++ b/ansible/roles/erwin-idm/templates/deployment.yml.j2 @@ -85,3 +85,40 @@ spec: timeoutSeconds: 4 failureThreshold: 30 periodSeconds: 10 +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 9 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: +{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %} + - weight: 10 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - erwinidm + topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }} +{% endif %} + - weight: 20 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - erwinidm + topologyKey: "topology.kubernetes.io/zone" +{% endif %} diff --git a/ansible/roles/hydra/templates/deployment.yml.j2 b/ansible/roles/hydra/templates/deployment.yml.j2 index 6972ba9cb..58330eb91 100644 --- a/ansible/roles/hydra/templates/deployment.yml.j2 +++ b/ansible/roles/hydra/templates/deployment.yml.j2 @@ -68,3 +68,40 @@ spec: requests: cpu: "{{ HYDRA_CPU_MIN|default("100m", true) }}" memory: "{{ HYDRA_MEM_MIN|default("128Mi", true) }}" +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 9 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: +{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %} + - weight: 10 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - hydra + topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }} +{% endif %} + - weight: 20 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - hydra + topologyKey: "topology.kubernetes.io/zone" +{% endif %} diff --git a/ansible/roles/maildrop/templates/deployment.yml.j2 b/ansible/roles/maildrop/templates/deployment.yml.j2 index 24ce47eeb..884374c36 100644 --- a/ansible/roles/maildrop/templates/deployment.yml.j2 +++ b/ansible/roles/maildrop/templates/deployment.yml.j2 @@ -61,3 +61,18 @@ spec: requests: cpu: {{ MAILDROP_CPU_REQUESTS|default("100m", true) }} memory: {{ MAILDROP_MEMORY_REQUESTS|default("128Mi", true) }} +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} +{% endif %} diff --git a/ansible/roles/oidcmock/templates/deployment.yml.j2 b/ansible/roles/oidcmock/templates/deployment.yml.j2 index 2d94a76ed..113575295 100644 --- a/ansible/roles/oidcmock/templates/deployment.yml.j2 +++ b/ansible/roles/oidcmock/templates/deployment.yml.j2 @@ -85,3 +85,18 @@ spec: path: clientsConfigurationContent.json - name: config-directory emptyDir: {} +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} +{% endif %} diff --git a/ansible/roles/rocketchat/templates/deployment.yml.j2 b/ansible/roles/rocketchat/templates/deployment.yml.j2 index 224dca4f7..485486e84 100644 --- a/ansible/roles/rocketchat/templates/deployment.yml.j2 +++ b/ansible/roles/rocketchat/templates/deployment.yml.j2 @@ -79,3 +79,40 @@ spec: volumes: - name: rocketchat-uploads-data emptyDir: {} +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 9 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: +{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %} + - weight: 10 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - rocketchat + topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }} +{% endif %} + - weight: 20 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - rocketchat + topologyKey: "topology.kubernetes.io/zone" +{% endif %} diff --git a/ansible/roles/rocketchat/templates/fixup-job.yml.j2 b/ansible/roles/rocketchat/templates/fixup-job.yml.j2 index 7ae963325..dd33666f4 100644 --- a/ansible/roles/rocketchat/templates/fixup-job.yml.j2 +++ b/ansible/roles/rocketchat/templates/fixup-job.yml.j2 @@ -47,6 +47,21 @@ spec: - key: update.sh path: update.sh restartPolicy: Never +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} +{% endif %} metadata: labels: app: rocketchat diff --git a/ansible/roles/storage/templates/deployment.yml.j2 b/ansible/roles/storage/templates/deployment.yml.j2 index db4a153d0..dc3e18437 100644 --- a/ansible/roles/storage/templates/deployment.yml.j2 +++ b/ansible/roles/storage/templates/deployment.yml.j2 @@ -66,3 +66,40 @@ spec: - name: storage-pv persistentVolumeClaim: claimName: storage-pvc +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 9 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: +{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %} + - weight: 10 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - minio + topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }} +{% endif %} + - weight: 20 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - minio + topologyKey: "topology.kubernetes.io/zone" +{% endif %}