Merge pull request #5105 from sgibson91/jmte/dedicated-nodegroups

jmte: create hub-specific nodegroups
2i2c-org · Nov 15, 2024 · 6e4c6e8 · 6e4c6e8
2 parents 7cfd9c9 + 46cce04
commit 6e4c6e8
Show file tree

Hide file tree

Showing 3 changed files with 144 additions and 11 deletions.
diff --git a/config/clusters/jupyter-meets-the-earth/prod.values.yaml b/config/clusters/jupyter-meets-the-earth/prod.values.yaml
@@ -30,4 +30,5 @@ basehub:
         SCRATCH_BUCKET: s3://jmte-scratch/$(JUPYTERHUB_USER)
         PANGEO_SCRATCH: s3://jmte-scratch/$(JUPYTERHUB_USER)
       nodeSelector:
+        2i2c/hub-name: prod
         node.kubernetes.io/instance-type: m5.xlarge
diff --git a/config/clusters/jupyter-meets-the-earth/staging.values.yaml b/config/clusters/jupyter-meets-the-earth/staging.values.yaml
@@ -13,5 +13,7 @@ basehub:
         - hosts: [staging.jmte.2i2c.cloud]
           secretName: https-auto-tls
     singleuser:
+      nodeSelector:
+        2i2c/hub-name: staging
       extraEnv:
         SCRATCH_BUCKET: s3://jupyter-meets-the-earth-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/eksctl/jupyter-meets-the-earth.jsonnet b/eksctl/jupyter-meets-the-earth.jsonnet
@@ -25,37 +25,165 @@ local nodeAz = "us-west-2a";
 // A `node.kubernetes.io/instance-type label is added, so pods
 // can request a particular kind of node with a nodeSelector
 local notebookNodes = [
-    { instanceType: "r5.xlarge" },
-    { instanceType: "r5.4xlarge" },
-    { instanceType: "r5.16xlarge" },
-    { instanceType: "m5.xlarge" },
-    { instanceType: "m5.4xlarge" },
-    { instanceType: "m5.16xlarge" },
-    { instanceType: "x1.16xlarge" },
+    { 
+        instanceType: "r5.xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    { 
+        instanceType: "r5.xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" },
+    },
+    { 
+        instanceType: "r5.4xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    { 
+        instanceType: "r5.4xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" },
+    },
+    { 
+        instanceType: "r5.16xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    { 
+        instanceType: "r5.16xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" },
+    },
+    { 
+        instanceType: "m5.xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    { 
+        instanceType: "m5.xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" },
+    },
+    { 
+        instanceType: "m5.4xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    { 
+        instanceType: "m5.4xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" },
+    },
+    { 
+        instanceType: "m5.16xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    { 
+        instanceType: "m5.16xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" },
+    },
+    { 
+        instanceType: "x1.16xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    { 
+        instanceType: "x1.16xlarge",
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" },
+    },
+    {
+        instanceType: "g4dn.xlarge",
+        minSize: 0,
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: {
+            "2i2c:hub-name": "staging",
+            "k8s.io/cluster-autoscaler/node-template/resources/nvidia.com/gpu": "1"
+        },
+        taints+: {
+          "nvidia.com/gpu": "NoSchedule"
+        }
+    },
     {
-        instanceType: "g4dn.xlarge", minSize: 0,
+        instanceType: "g4dn.xlarge",
+        minSize: 0,
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
         tags+: {
+            "2i2c:hub-name": "prod",
             "k8s.io/cluster-autoscaler/node-template/resources/nvidia.com/gpu": "1"
         },
         taints+: {
           "nvidia.com/gpu": "NoSchedule"
         }
     },
     {
-        instanceType: "g4dn.4xlarge", minSize: 0,
+        instanceType: "g4dn.4xlarge",
+        minSize: 0,
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
         tags+: {
+            "2i2c:hub-name": "staging",
             "k8s.io/cluster-autoscaler/node-template/resources/nvidia.com/gpu": "1"
         },
         taints+: {
           "nvidia.com/gpu": "NoSchedule"
         }
     },
     {
-        instanceType: "g4dn.16xlarge", minSize: 0,
+        instanceType: "g4dn.4xlarge",
+        minSize: 0,
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: {
+            "2i2c:hub-name": "prod",
+            "k8s.io/cluster-autoscaler/node-template/resources/nvidia.com/gpu": "1"
+        },
+        taints+: {
+          "nvidia.com/gpu": "NoSchedule"
+        }
+    },
+    {
+        instanceType: "g4dn.16xlarge",
+        minSize: 0,
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        taints+: {
+          "nvidia.com/gpu": "NoSchedule"
+        },
+        tags+: {
+            "2i2c:hub-name": "staging",
+            "k8s.io/cluster-autoscaler/node-template/resources/nvidia.com/gpu": "1"
+        }
+    },
+    {
+        instanceType: "g4dn.16xlarge",
+        minSize: 0,
+        namePrefix: "nb-prod",
+        labels+: { "2i2c/hub-name": "prod" },
         taints+: {
           "nvidia.com/gpu": "NoSchedule"
         },
         tags+: {
+            "2i2c:hub-name": "prod",
             "k8s.io/cluster-autoscaler/node-template/resources/nvidia.com/gpu": "1"
         }
     },
@@ -112,7 +240,7 @@ local daskNodes = [
     [
         ng + {
             namePrefix: 'core',
-            nameSuffix: 'a',
+            nameSuffix: 'b',
             nameIncludeInstanceType: false,
             availabilityZones: [nodeAz],
             ssh: {
@@ -125,6 +253,7 @@ local daskNodes = [
                 "hub.jupyter.org/node-purpose": "core",
                 "k8s.dask.org/node-purpose": "core"
             },
+            tags+: { "2i2c:node-purpose": "core" },
         },
     ] + [
         ng + {
@@ -140,6 +269,7 @@ local daskNodes = [
                 "hub.jupyter.org/node-purpose": "user",
                 "k8s.dask.org/node-purpose": "scheduler"
             },
+            tags+: { "2i2c:node-purpose": "user" },
             taints+: {
                 "hub.jupyter.org_dedicated": "user:NoSchedule",
                 "hub.jupyter.org/dedicated": "user:NoSchedule"