Set max resources in jobs

artifacts/k8s_galaxy.yml

@@ -4,25 +4,84 @@
   connection: local
   vars:
     NAMESPACE: galaxy
+    AFFINITY:
+      affinity:
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                topologyKey: "kubernetes.io/hostname"
+                labelSelector:
+                  matchExpressions:
+                    - key: app.kubernetes.io/instance
+                      operator: In
+                      values:
+                        - galaxy
+                        - postgres-operator
+                        - rabbitmq-operator
     admin_users: "{{ galaxy_admin_users | default('[email protected]') }}"
     brand: "{{ galaxy_brand | default('IM Deployed Galaxy K8s Cluster') }}"
+    max_mem: "{{ galaxy_max_mem | default(0) }}"
+    max_cores: "{{ galaxy_max_cores | default(0) }}"
   tasks:
-    - name: Install Git
-      package: name=git
-
-    - name: Clone galaxy-cvmfs-csi-helm git repo
-      git:
-        repo: https://github.com/CloudVE/galaxy-cvmfs-csi-helm
-        dest: /opt/galaxy-cvmfs-csi-helm
-        update: false
-        version: "master"
-
-    - name: Clone galaxy-helm git repo
-      git:
-        repo: https://github.com/galaxyproject/galaxy-helm
-        dest: /opt/galaxy-helm
-        update: false
-        version: "master"
+
+    - name: "Check if postgres-operator is already deployed"
+      command: helm status postgres-operator -n {{ NAMESPACE }}
+      register: deployed
+      failed_when: deployed.rc != 0 and deployed.rc != 1
+      changed_when: false
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+
+    - when: deployed.rc == 1
+      block:
+
+        - name: Create postgres-operator helm chart values file
+          copy:
+            dest: /opt/postgres_values.yaml
+            mode: '644'
+            content: "{{ AFFINITY | to_yaml }}"
+
+        - name: Add helm repo postgres-operator-charts
+          command: helm repo add postgres-operator-charts https://opensource.zalando.com/postgres-operator/charts/postgres-operator
+          environment:
+            KUBECONFIG: /etc/kubernetes/admin.conf
+
+        - name: Deploy postgres-operator
+          command: helm install -n {{ NAMESPACE }} --create-namespace postgres-operator postgres-operator-charts/postgres-operator --version 1.9.0 -f /opt/postgres_values.yaml --wait
+          environment:
+            KUBECONFIG: /etc/kubernetes/admin.conf
+
+    - name: "Check if rabbitmq-operator is already deployed"
+      command: helm status rabbitmq-operator -n {{ NAMESPACE }}
+      register: deployed
+      failed_when: deployed.rc != 0 and deployed.rc != 1
+      changed_when: false
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+
+    - when: deployed.rc == 1
+      block:
+
+        - name: Create rabbitmq-operator helm chart values file
+          copy:
+            dest: /opt/rabbitmq_values.yaml
+            mode: '644'
+            content: |
+              clusterOperator:
+                {{ AFFINITY | to_yaml(indent=4) }}
+              msgTopologyOperator:
+                {{ AFFINITY | to_yaml(indent=4) }}
+
+        - name: Add helm repo bitnami
+          command: helm repo add bitnami https://charts.bitnami.com/bitnami
+          environment:
+            KUBECONFIG: /etc/kubernetes/admin.conf
+
+        - name: Deploy rabbitmq-operator
+          command: helm install -n {{ NAMESPACE }} --create-namespace rabbitmq-operator bitnami/rabbitmq-cluster-operator --version 2.6.12 -f /opt/rabbitmq_values.yaml
+          environment:
+            KUBECONFIG: /etc/kubernetes/admin.conf
 
     - name: "Check if galaxy-cvmfs is already deployed"
       command: helm status galaxy-cvmfs -n {{ NAMESPACE }}
@@ -35,18 +94,31 @@
     - when: deployed.rc == 1
       block:
 
-        - name: Update Galaxy CVMFS CSI chart dependencies
-          command: helm dep up
-          args:
-            chdir: /opt/galaxy-cvmfs-csi-helm/galaxy-cvmfs-csi
-            creates: /opt/galaxy-cvmfs-csi-helm/galaxy-cvmfs-csi/charts
+        - name: Create galaxy helm chart values file
+          copy:
+            dest: /opt/galaxy-cvmfs-csi_values.yaml
+            mode: '644'
+            content: |
+              storageClassName: galaxy-cvmfs
+              cvmfscsi:
+                cache:
+                  alien:
+                    pvc:
+                      storageClass: managed-nfs-storage
+                      name: cvmfs-alien-cache
+                # try to pack all the pods in the same nodes
+                controllerplugin:
+                  {{ AFFINITY | to_yaml(indent=4) }}
+                nodeplugin:
+                  {{ AFFINITY | to_yaml(indent=4) }}
+
+        - name: Add helm repo galaxy
+          command: helm repo add galaxy https://raw.githubusercontent.com/CloudVE/helm-charts/master/
           environment:
             KUBECONFIG: /etc/kubernetes/admin.conf
 
         - name: Deploy Galaxy CVMFS CSI
-          command: helm install --create-namespace -n {{ NAMESPACE }} galaxy-cvmfs . --wait --set storageClassName="galaxy-cvmfs" --set cvmfscsi.cache.alien.pvc.storageClass=managed-nfs-storage --set cvmfscsi.cache.alien.pvc.name="cvmfs-alien-cache"
-          args:
-            chdir: /opt/galaxy-cvmfs-csi-helm/galaxy-cvmfs-csi
+          command: helm install --create-namespace -n {{ NAMESPACE }} galaxy-cvmfs galaxy/galaxy-cvmfs-csi --wait -f  /opt/galaxy-cvmfs-csi_values.yaml --version 2.2.0
           environment:
             KUBECONFIG: /etc/kubernetes/admin.conf
 
@@ -61,13 +133,10 @@
     - when: deployed.rc == 1
       block:
 
-        - name: Update Galaxy chart dependencies
-          command: helm dep up
-          args:
-            chdir: /opt/galaxy-helm/galaxy
-            creates: /opt/galaxy-helm/galaxy/charts
-          environment:
-            KUBECONFIG: /etc/kubernetes/admin.conf
+        - name: Remove GB from max_mem
+          set_fact:
+            max_mem: "{{ max_mem[:-2] | float - 1 }}"
+          when: max_mem and "GB" in max_mem
 
         - name: Create galaxy helm chart values file
           copy:
@@ -79,18 +148,36 @@
               cvmfs:
                 deploy: false
                 storageClassName: galaxy-cvmfs
+              postgresql:
+                deploy: false
+              rabbitmq:
+                deploy: false
               configs:
                 galaxy.yml:
                   galaxy:
                     admin_users: {{ admin_users }}
                     brand: {{ brand }}
+              jobs:
+                rules:
+                  tpv_rules_local.yml:
+                    destinations:
+                      k8s:
+                        runner: k8s
+                        {{ 'max_cores: ' ~ (max_cores | int - 1) if max_cores | int > 0 else '# no max_cores' }}
+                        {{ 'max_mem: ' ~ max_mem if max_mem | int > 0 else '# no max_mem' }}
+              # try to pack all the pods in the same nodes
+              {{ AFFINITY | to_yaml }}
+
+        - name: Add helm repo galaxy
+          command: helm repo add galaxy https://raw.githubusercontent.com/CloudVE/helm-charts/master/
+          environment:
+            KUBECONFIG: /etc/kubernetes/admin.conf
 
         - name: Deploy Galaxy
-          command: helm install --create-namespace -n {{ NAMESPACE }} galaxy . --timeout 15m -f /opt/galaxy_values.yaml
-          args:
-            chdir: /opt/galaxy-helm/galaxy
+          command: helm install --create-namespace -n {{ NAMESPACE }} galaxy galaxy/galaxy --timeout 10m -f /opt/galaxy_values.yaml --version 5.9.0
           environment:
             KUBECONFIG: /etc/kubernetes/admin.conf
+          ignore_errors: true
 
         - name: Wait pods to be ready
           pause:

templates/k8s_galaxy.yaml

@@ -37,8 +37,8 @@ topology_template:
         - valid_values: [ docker ]
     wn_num:
       type: integer
-      description: Number of WNs in the cluster
-      default: 2
+      description: "Number of WNs in the cluster (Minimum recommended: 3)"
+      default: 3
       required: yes
     wn_cpus:
       type: integer
@@ -70,6 +70,8 @@ topology_template:
             inputs:
               galaxy_admin_users: { get_input: galaxy_admin_users }
               galaxy_brand: { get_input: galaxy_brand }
+              galaxy_max_mem: { get_input: wn_mem }
+              galaxy_max_cores: { get_input: wn_cpus }
 
   outputs:
     galaxy_endpoint: