BC-5423 - Add POD affinity and anti-affinity (#4606)

To Improve the distribution of PODs to the least number of kubernetes worker nodes who is Required but also to secure that the PODs of one Deployment are distributed over the Fault Zones and Node pools the pod affinity and anti affinity are set. Affernety number calculation: Assign Number for an host = 0 - 20 X Fault Zone - 10 X Node Pool + 9 X Host Host = 1 if on the host runs an POD from the Schulcloud-Verbund on any namespace Host = 0 if on the host runs no POD from the Schulcloud-Verbund on any namespace Node Pool = 1 if on a host of this node pool runs an POD from the Deployment Node Pool = 0 if on no host of this node pool runs an POD from the Deployment Fault Pool = 1 if on a host of this fault zone runs an POD from the Deployment Fault Pool = 0 if on no host of this fault zone uns an POD from the Deployment
hpi-schul-cloud · Dec 4, 2023 · 23aff06 · 23aff06
1 parent ece3f0d
commit 23aff06
Show file tree

Hide file tree

Showing 17 changed files with 517 additions and 1 deletion.
diff --git a/ansible/roles/h5p-library-management/templates/api-h5p-library-management-cronjob.yml.j2 b/ansible/roles/h5p-library-management/templates/api-h5p-library-management-cronjob.yml.j2
@@ -4,6 +4,13 @@ metadata:
   namespace: {{ NAMESPACE }}
   labels:
     app: api-library-management-cronjob
+    app.kubernetes.io/part-of: schulcloud-verbund
+    app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+    app.kubernetes.io/name:  api-library-management-cronjob
+    app.kubernetes.io/component: h5p
+    app.kubernetes.io/managed-by: ansible
+    git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }}
+    git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }}
   name: api-library-management-cronjob
 spec:
   schedule: "{{ SERVER_H5P_LIBRARY_MANAGEMENT_CRONJOB|default("0 3 * * 3,6", true) }}"
@@ -12,6 +19,16 @@ spec:
     spec:
       activeDeadlineSeconds: {{ SERVER_H5P_LIBRARY_MANAGEMENT_CRONJOB_TIMEOUT|default("39600", true) }}
       template:
+        metadata:
+          labels:
+            app:  api-library-management-cronjob
+            app.kubernetes.io/part-of: schulcloud-verbund
+            app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+            app.kubernetes.io/name:  api-library-management-cronjob
+            app.kubernetes.io/component: h5p
+            app.kubernetes.io/managed-by: ansible
+            git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }}
+            git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }}
         spec:
           containers:
           - name: api-h5p-library-management-cronjob
@@ -31,3 +48,18 @@ spec:
                 cpu: {{ API_H5P_LIBRARY_MANAGEMENT_CPU_REQUESTS|default("100m", true) }}
                 memory: {{ API_H5P_LIBRARY_MANAGEMENT_MEMORY_REQUESTS|default("150Mi", true) }}
           restartPolicy: OnFailure
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+          affinity:
+            podAffinity:
+              preferredDuringSchedulingIgnoredDuringExecution:
+              - weight: 100
+                podAffinityTerm:
+                  labelSelector:
+                    matchExpressions:
+                    - key: app.kubernetes.io/part-of
+                      operator: In
+                      values:
+                      - schulcloud-verbund
+                  topologyKey: "kubernetes.io/hostname"
+                  namespaceSelector: {}
+{% endif %}
diff --git a/ansible/roles/schulcloud-server-core/templates/admin-api-server-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/admin-api-server-deployment.yml.j2
@@ -61,4 +61,41 @@ spec:
             memory: {{ ADMIN_API_SERVER_MEMORY_LIMITS|default("4Gi", true) }}
           requests:
             cpu: {{  ADMIN_API_SERVER_CPU_REQUESTS|default("100m", true) }}
-            memory: {{  ADMIN_API_SERVER_MEMORY_REQUESTS|default("150Mi", true) }}
+            memory: {{  ADMIN_API_SERVER_MEMORY_REQUESTS|default("150Mi", true) }}
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+      affinity:
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 9
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/part-of
+                  operator: In
+                  values:
+                  - schulcloud-verbund
+              topologyKey: "kubernetes.io/hostname"
+              namespaceSelector: {}
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %}
+          - weight: 10
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - api-admin
+              topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }}
+{% endif %}
+          - weight: 20
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - api-admin
+              topologyKey: "topology.kubernetes.io/zone"
+{% endif %}
diff --git a/ansible/roles/schulcloud-server-core/templates/amqp-files-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/amqp-files-deployment.yml.j2
@@ -58,3 +58,42 @@ spec:
           requests:
             cpu: {{ AMQP_FILE_STORAGE_CPU_REQUESTS|default("100m", true) }}
             memory: {{ AMQP_FILE_STORAGE_MEMORY_REQUESTS|default("50Mi", true) }}
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+      affinity:
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 9
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/part-of
+                  operator: In
+                  values:
+                  - schulcloud-verbund
+              topologyKey: "kubernetes.io/hostname"
+              namespaceSelector: {}
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %}
+          - weight: 10
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - amqp-files
+              topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }}
+{% endif %}
+          - weight: 20
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - amqp-files
+              topologyKey: "topology.kubernetes.io/zone"
+{% endif %}
+
+
diff --git a/ansible/roles/schulcloud-server-core/templates/api-delete-s3-files-cronjob.yml.j2 b/ansible/roles/schulcloud-server-core/templates/api-delete-s3-files-cronjob.yml.j2
@@ -38,6 +38,21 @@ spec:
                 cpu: {{ API_CPU_REQUESTS|default("100m", true) }}
                 memory: {{ API_MEMORY_REQUESTS|default("150Mi", true) }}
           restartPolicy: OnFailure
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+          affinity:
+            podAffinity:
+              preferredDuringSchedulingIgnoredDuringExecution:
+              - weight: 100
+                podAffinityTerm:
+                  labelSelector:
+                    matchExpressions:
+                    - key: app.kubernetes.io/part-of
+                      operator: In
+                      values:
+                      - schulcloud-verbund
+                  topologyKey: "kubernetes.io/hostname"
+                  namespaceSelector: {}
+{% endif %}
         metadata:
           labels:
             app: api

diff --git a/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2
@@ -84,3 +84,40 @@ spec:
           requests:
             cpu: {{ API_FILE_STORAGE_CPU_REQUESTS|default("100m", true) }}
             memory: {{ API_FILE_STORAGE_MEMORY_REQUESTS|default("50Mi", true) }}
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+      affinity:
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 9
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/part-of
+                  operator: In
+                  values:
+                  - schulcloud-verbund
+              topologyKey: "kubernetes.io/hostname"
+              namespaceSelector: {}
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %}
+          - weight: 10
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - api-files
+              topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }}
+{% endif %}
+          - weight: 20
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - api-files
+              topologyKey: "topology.kubernetes.io/zone"
+{% endif %}
diff --git a/ansible/roles/schulcloud-server-core/templates/api-fwu-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/api-fwu-deployment.yml.j2
@@ -84,3 +84,40 @@ spec:
           requests:
             cpu: {{ API_FWU_LEARNING_CONTENTS_CPU_REQUESTS|default("100m", true) }}
             memory: {{ API_FWU_LEARNING_CONTENTS_MEMORY_REQUESTS|default("50Mi", true) }}
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+      affinity:
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 9
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/part-of
+                  operator: In
+                  values:
+                  - schulcloud-verbund
+              topologyKey: "kubernetes.io/hostname"
+              namespaceSelector: {}
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %}
+          - weight: 10
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - api-fwu
+              topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }}
+{% endif %}
+          - weight: 20
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - api-fwu
+              topologyKey: "topology.kubernetes.io/zone"
+{% endif %}
diff --git a/ansible/roles/schulcloud-server-core/templates/data-deletion-trigger-cronjob.yml.j2 b/ansible/roles/schulcloud-server-core/templates/data-deletion-trigger-cronjob.yml.j2
@@ -45,6 +45,21 @@ spec:
                 cpu: {{ API_CPU_REQUESTS|default("100m", true) }}
                 memory: {{ API_MEMORY_REQUESTS|default("150Mi", true) }}
           restartPolicy: OnFailure
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+          affinity:
+            podAffinity:
+              preferredDuringSchedulingIgnoredDuringExecution:
+              - weight: 100
+                podAffinityTerm:
+                  labelSelector:
+                    matchExpressions:
+                    - key: app.kubernetes.io/part-of
+                      operator: In
+                      values:
+                      - schulcloud-verbund
+                  topologyKey: "kubernetes.io/hostname"
+                  namespaceSelector: {}
+{% endif %}
         metadata:
           labels:
             app: data-deletion-trigger

diff --git a/ansible/roles/schulcloud-server-core/templates/deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/deployment.yml.j2
@@ -86,3 +86,40 @@ spec:
           requests:
             cpu: {{ API_CPU_REQUESTS|default("100m", true) }}
             memory: {{ API_MEMORY_REQUESTS|default("150Mi", true) }}
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+      affinity:
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 9
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/part-of
+                  operator: In
+                  values:
+                  - schulcloud-verbund
+              topologyKey: "kubernetes.io/hostname"
+              namespaceSelector: {}
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %}
+          - weight: 10
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - api
+              topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }}
+{% endif %}
+          - weight: 20
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - api
+              topologyKey: "topology.kubernetes.io/zone"
+{% endif %}
diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2
@@ -58,3 +58,40 @@ spec:
           requests:
             cpu: {{ AMQP_FILE_PREVIEW_CPU_REQUESTS|default("100m", true) }}
             memory: {{ AMQP_FILE_PREVIEW_MEMORY_REQUESTS|default("250Mi", true) }}
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+      affinity:
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 9
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/part-of
+                  operator: In
+                  values:
+                  - schulcloud-verbund
+              topologyKey: "kubernetes.io/hostname"
+              namespaceSelector: {}
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %}
+          - weight: 10
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - preview-generator
+              topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }}
+{% endif %}
+          - weight: 20
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - preview-generator
+              topologyKey: "topology.kubernetes.io/zone"
+{% endif %}
diff --git a/ansible/roles/schulcloud-server-core/templates/tldraw-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/tldraw-deployment.yml.j2
@@ -65,3 +65,40 @@ spec:
           requests:
             cpu: {{ TLDRAW_EDITOR_CPU_REQUESTS|default("100m", true) }}
             memory: {{ TLDRAW_EDITOR_MEMORY_REQUESTS|default("150Mi", true) }}
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+      affinity:
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 9
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/part-of
+                  operator: In
+                  values:
+                  - schulcloud-verbund
+              topologyKey: "kubernetes.io/hostname"
+              namespaceSelector: {}
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %}
+          - weight: 10
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - tldraw-server
+              topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }}
+{% endif %}
+          - weight: 20
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - tldraw-server
+              topologyKey: "topology.kubernetes.io/zone"
+{% endif %}