Merge pull request #5124 from sgibson91/opensci/dedicated-nodegroups

opensci: create hub-specific nodegroups
2i2c-org · Nov 15, 2024 · a407651 · a407651
2 parents 2f69308 + e06ef81
commit a407651
Show file tree

Hide file tree

Showing 6 changed files with 111 additions and 4 deletions.
diff --git a/config/clusters/opensci/big-binder.values.yaml b/config/clusters/opensci/big-binder.values.yaml
@@ -52,6 +52,7 @@ jupyterhub:
       # Schedule users on the smallest instance
       # https://github.com/2i2c-org/infrastructure/issues/4241
       node.kubernetes.io/instance-type: r5.xlarge
+      2i2c/hub-name: big-binder
     cloudMetadata:
       blockWithIptables: false
     cpu:
@@ -100,6 +101,7 @@ binderhub-service:
       # Schedule dockerApi pods to run on the smallest user nodes only
       # https://github.com/2i2c-org/infrastructure/issues/4241
       node.kubernetes.io/instance-type: r5.xlarge
+      2i2c/hub-name: big-binder
   ingress:
     enabled: true
     hosts: [big.binder.opensci.2i2c.cloud]
@@ -114,6 +116,7 @@ binderhub-service:
         # Schedule builder pods to run on the smallest user nodes only
         # https://github.com/2i2c-org/infrastructure/issues/4241
         node.kubernetes.io/instance-type: r5.xlarge
+        2i2c/hub-name: big-binder
     BinderHub:
       base_url: /
       hub_url: https://hub.big.binder.opensci.2i2c.cloud

diff --git a/config/clusters/opensci/climaterisk.values.yaml b/config/clusters/opensci/climaterisk.values.yaml
@@ -32,6 +32,7 @@ jupyterhub:
       tag: "09c3a11f1698"
     nodeSelector:
       node.kubernetes.io/instance-type: r5.4xlarge
+      2i2c/hub-name: climaterisk
     memory:
       guarantee: 15929958203
       limit: 15929958203

diff --git a/config/clusters/opensci/sciencecore.values.yaml b/config/clusters/opensci/sciencecore.values.yaml
@@ -39,6 +39,8 @@ jupyterhub:
     singleuserAdmin:
       serviceAccountName: admin-sa
   singleuser:
+    nodeSelector:
+      2i2c/hub-name: sciencecore
     defaultUrl: /lab
     extraEnv:
       SCRATCH_BUCKET: s3://opensci-scratch-sciencecore/$(JUPYTERHUB_USER)

diff --git a/config/clusters/opensci/small-binder.values.yaml b/config/clusters/opensci/small-binder.values.yaml
@@ -21,6 +21,7 @@ jupyterhub:
       # Schedule users on the smallest instance
       # https://github.com/2i2c-org/infrastructure/issues/4241
       node.kubernetes.io/instance-type: r5.xlarge
+      2i2c/hub-name: small-binder
     cpu:
       limit: 2
     memory:
@@ -86,6 +87,7 @@ binderhub-service:
       # Schedule dockerApi pods to run on the smallest user nodes only
       # https://github.com/2i2c-org/infrastructure/issues/4241
       node.kubernetes.io/instance-type: r5.xlarge
+      2i2c/hub-name: small-binder
   ingress:
     enabled: true
     hosts: [binder.opensci.2i2c.cloud]
@@ -100,6 +102,7 @@ binderhub-service:
         # Schedule builder pods to run on the smallest user nodes only
         # https://github.com/2i2c-org/infrastructure/issues/4241
         node.kubernetes.io/instance-type: r5.xlarge
+        2i2c/hub-name: small-binder
     BinderHub:
       base_url: /
       hub_url: https://hub.binder.opensci.2i2c.cloud

diff --git a/config/clusters/opensci/staging.values.yaml b/config/clusters/opensci/staging.values.yaml
@@ -28,6 +28,8 @@ jupyterhub:
           name: ""
           url: ""
   singleuser:
+    nodeSelector:
+      2i2c/hub-name: staging
     profileList:
       - display_name: Choose your environment and resources
         slug: only-choice

diff --git a/eksctl/opensci.jsonnet b/eksctl/opensci.jsonnet
@@ -25,9 +25,103 @@ local nodeAz = "us-west-2a";
 // A `node.kubernetes.io/instance-type label is added, so pods
 // can request a particular kind of node with a nodeSelector
 local notebookNodes = [
+    // FIXME: tainted, to be deleted when empty, replaced by equivalent
     { instanceType: "r5.xlarge" },
-    { instanceType: "r5.4xlarge" },
-    { instanceType: "r5.16xlarge" },
+    // staging hub
+    {
+        instanceType: "r5.xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    {
+        instanceType: "r5.4xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    {
+        instanceType: "r5.16xlarge",
+        namePrefix: "nb-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+    },
+    // sciencecore hub
+    {
+        instanceType: "r5.xlarge",
+        namePrefix: "nb-sciencecore",
+        labels+: { "2i2c/hub-name": "sciencecore" },
+        tags+: { "2i2c:hub-name": "sciencecore" },
+    },
+    {
+        instanceType: "r5.4xlarge",
+        namePrefix: "nb-sciencecore",
+        labels+: { "2i2c/hub-name": "sciencecore" },
+        tags+: { "2i2c:hub-name": "sciencecore" },
+    },
+    {
+        instanceType: "r5.16xlarge",
+        namePrefix: "nb-sciencecore",
+        labels+: { "2i2c/hub-name": "sciencecore" },
+        tags+: { "2i2c:hub-name": "sciencecore" },
+    },
+    // climaterisk hub
+    {
+        instanceType: "r5.xlarge",
+        namePrefix: "nb-climaterisk",
+        labels+: { "2i2c/hub-name": "climaterisk" },
+        tags+: { "2i2c:hub-name": "climaterisk" },
+    },
+    {
+        instanceType: "r5.4xlarge",
+        namePrefix: "nb-climaterisk",
+        labels+: { "2i2c/hub-name": "climaterisk" },
+        tags+: { "2i2c:hub-name": "climaterisk" },
+    },
+    {
+        instanceType: "r5.16xlarge",
+        namePrefix: "nb-climaterisk",
+        labels+: { "2i2c/hub-name": "climaterisk" },
+        tags+: { "2i2c:hub-name": "climaterisk" },
+    },
+    // small-binder hub
+    {
+        instanceType: "r5.xlarge",
+        namePrefix: "nb-small-binder",
+        labels+: { "2i2c/hub-name": "small-binder" },
+        tags+: { "2i2c:hub-name": "small-binder" },
+    },
+    {
+        instanceType: "r5.4xlarge",
+        namePrefix: "nb-small-binder",
+        labels+: { "2i2c/hub-name": "small-binder" },
+        tags+: { "2i2c:hub-name": "small-binder" },
+    },
+    {
+        instanceType: "r5.16xlarge",
+        namePrefix: "nb-small-binder",
+        labels+: { "2i2c/hub-name": "small-binder" },
+        tags+: { "2i2c:hub-name": "small-binder" },
+    },
+    // big-binder hub
+    {
+        instanceType: "r5.xlarge",
+        namePrefix: "nb-big-binder",
+        labels+: { "2i2c/hub-name": "big-binder" },
+        tags+: { "2i2c:hub-name": "big-binder" },
+    },
+    {
+        instanceType: "r5.4xlarge",
+        namePrefix: "nb-big-binder",
+        labels+: { "2i2c/hub-name": "big-binder" },
+        tags+: { "2i2c:hub-name": "big-binder" },
+    },
+    {
+        instanceType: "r5.16xlarge",
+        namePrefix: "nb-big-binder",
+        labels+: { "2i2c/hub-name": "big-binder" },
+        tags+: { "2i2c:hub-name": "big-binder" },
+    },
 ];
 local daskNodes = [
     // Node definitions for dask worker nodes. Config here is merged
@@ -50,7 +144,7 @@ local daskNodes = [
     metadata+: {
         name: "opensci",
         region: clusterRegion,
-        version: "1.29",
+        version: "1.30",
     },
     availabilityZones: masterAzs,
     iam: {
@@ -81,7 +175,7 @@ local daskNodes = [
     [
         ng + {
             namePrefix: 'core',
-            nameSuffix: 'b',
+            nameSuffix: 'a',
             nameIncludeInstanceType: false,
             availabilityZones: [nodeAz],
             ssh: {
@@ -94,6 +188,7 @@ local daskNodes = [
                 "hub.jupyter.org/node-purpose": "core",
                 "k8s.dask.org/node-purpose": "core"
             },
+            tags+: { "2i2c:node-purpose": "core" },
         },
     ] + [
         ng + {
@@ -109,6 +204,7 @@ local daskNodes = [
                 "hub.jupyter.org/node-purpose": "user",
                 "k8s.dask.org/node-purpose": "scheduler"
             },
+            tags+: { "2i2c:node-purpose": "user" },
             taints+: {
                 "hub.jupyter.org_dedicated": "user:NoSchedule",
                 "hub.jupyter.org/dedicated": "user:NoSchedule"