--wip-- [skipci]

LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2024 Rare Compute Foundation, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

kubernetes/arc1/apps/dev/gha-runner-scale-set-controller/app/helmrelease.yaml

@@ -0,0 +1,25 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: &app gha-runner-scale-set-controller
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: gha-runner-scale-set-controller
+      version: 0.9.3
+      sourceRef:
+        kind: HelmRepository
+        name: actions-runner-controller
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  values:
+    fullnameOverride: *app

kubernetes/arc1/apps/dev/gha-runner-scale-set-controller/app/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./secret.sops.yaml
+  - ./helmrelease.yaml

kubernetes/arc1/apps/dev/gha-runner-scale-set-controller/app/secret.sops.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: actions-runner-controller-secret
+  namespace: action-runner-system
+stringData:
+  github_app_id: ENC[AES256_GCM,data:U9S/PHEyuQ==,iv:X0pb8ZFRCfUX1ERQ7ECcTx+MEl5mNWo8VILVmeY4KGE=,tag:8rTatI+eB+WVykcyCsIwfw==,type:int]
+  github_app_installation_id: ENC[AES256_GCM,data:fc1Unmxc/SA=,iv:+RAGj81+wBaOaadEoBz4nVtL0E7k4cg9IQZQU4mpEjw=,tag:ZlUlZDYPqAqAVn/kpecchA==,type:int]
+  github_app_private_key: ENC[AES256_GCM,data:ht73CFpKOd7NQLxvMI2SBO5kwBlNKp35LKOF7/5RVuCVi/kPpec9QNiIjfMx1AwhCQXYgY1v9WSCCdWAvIZrDpf7xMsWdnais/1VhRxppVNJRhaR3Vg/zrWdbK53PPDx9f5zZzAxyZ5XF5sDJsEByM490T+exUxNss9AoxbkYvMcsdG6FWNNTvl69d/orC7pv8Gb/r5x2EFCBcg67yeu22ffuXqpEJUWiL85QPRTPQduuHKvI0wmcT68285A8Hzla6RbpaHdQSgVShfypQphpU+394RxhBb1m7tSd73mvQV7g8mbcCXvag+so0TU/NhkGjyRdYx/81z8cYoqBmss5FD69nHsLI6soirkDJSw27CVNOHXCNrFSYwKGJ6Sqmbdop6oA8EVtcxGgcPBZK59i8AjhWiyybwmPtDLWPbiWWj5hwCIDzcftlkuChp61R914rDo7wY4WNxq8/56rH164TxzPYSzZCNSzrle/ub+gzvJzBlWlNiiTF20r5q9HqU3Yq3STK3ouunAahHuPxm/B/sJwv1Yprt6VTCJsRNexTC0/ROqdhD3/La0QrRAslFxbUbfpLxzn3VaYY01B/nVI9uQBVcbWz2ziXT/v1zU7Enk38V2cRJUoFZ6eU5yuwc9Pw7rqFeAjeZSRz0wA4vb6+lbKjXD3dM9gHhjowT6R6E/DnIVZbE8mPn2x8rsWYkfeFQ4Y24JAkgvrulFW8OOm5JYaGuYzj61Qcy3e8WKSkTB8YMoUkNGMGb8J/LQqgi8qlWRe00ytZmC6HzKsWRrvni7/i2Xqeju6dXB+mKmoKShQ2APHg90yRY+TH2Nl342knrm5EcqKhW3f7iszTxdqpSs05hO8q9nr5T5EQXxrQhRG5bcD1zmISAqJoo1Ad6luHv6DzNwk673woEQ/EayvPHeL7CUzrBjLgY/fwgGgwPAvqmudF44nBdDdsGsF0y2QstmLujQnRZi+z2tFYr8OZmdkhSfd4PznTeydOkp9Fif2O6U8+IKATCv850smZkRvWtMr+j63VnpVcf6YupMiU9SYGC+jFMNr7HYcbbXS6SnIdQlgVGqx4c645t7KRluC/FR70D1s//Dya62ncC89V6vzyf3C4JPiTj6a/ftM0NjZvNv0m2Lgb4HAr4Voj/wIT7XLAfEMLGO1gz52+/nAhtJt59NRxZuMXFYljJHlbkpWC1KivQxy/miEkX0jeL/P0ujIPCGorDUaRUvdhifVunSz1L5+E0/4nlCWxCNdXVIoGnVFz2M89RZ16Tcl82BtSa06c2JEh0vObAuKAuzTkEIUe00f9CP4QRwaj9AwBZ/ueCvEoacBN4F5TLKZ+AffgIz/y1yuAhO5eICCBBhoEVnIHGfDGcmrg5rwtUkPRUHA9pwihZ9Qrok6Bkj3lgsSnle6kpb0l/nEsBssAI0sAKNaGsc5adu3KcIdsqVuMLk87WQD7m9XVa0o/+NTy6FgMb/Spe39QeUUkU8W2G0cX+/sINaA65BruGcT719Z5QW+/VsiSXwujQCaW3tqoEH7HmwYC7wKqki4vboL7oUxm/JAhaOsll1Yj4Ui9mkRYflkGoAHBz3vkW+naL70it+son9IM/2ZsMrjduqXocn5wuoG8YMLQlxznn4D2JncQr8Up4qmQa8/UkQhXDpE5+/lCtGvW2ptgULBm+U/4mLHT9W9Xm4Qk9PJj0eFw7Ytk/doHO7BaPMe+fCVv9faty8qUatI8yG7cmbi7+reZXw1R6FozfUKhfl9MCbjXlc0x9iah5NT53Z0AE7USwfL7oHBG76q/rJJM+mVXWoprXzkMU9PP2fv/EzQVW5pN4XXPcz+E9jzmoCBcTCz+8qmMUJ7llfaeSH3RJ3/tPdmlWUnR5rgEXs5C4qrSvJ8wYvZXAvVMPhCzLwCMgbZCWA5ox0h0etOBpoHqBLOQyR12xDgWMGP4T3b+KHM+Yv6u4OaXHlDH1A9yZVKk0IO/3jNbZoJ+P9g4fb6teN5dvvdOA9Fg3vmks5o5Vrfx6HTJ0eLfWPDuSJ6Ig1dvIRm7i+9iDi8H8G8YhpEZpbayjasbnpI7CHYwnKmqmFyLsMKZmaP1N4Gro3sKlrvtFmSq0XDJOH3DYgHfL/30xYGtdNnls2miUkhS4byF4flxZM9ux2e6iTWVqJx/10Js6gon9hZqouDIerRtzCO3KwFZwBblLTlNlgXCUW74NWXKy3EOBBhbg4CiKkBRVSHoorYobcc/E=,iv:FK77AOqlpI2WO5rV+S2p/KkpiCiWo4qUNoixx69Kmww=,tag:LcRuneNb9JptghePcnDMbQ==,type:str]
+sops:
+  kms: []
+  gcp_kms: []
+  azure_kv: []
+  hc_vault: []
+  age:
+    - recipient: age1ey3reuxyffqynll464r4q3tlhq5v73nxesyktr44lfez8jzxm94s0644n7
+      enc: |
+        -----BEGIN AGE ENCRYPTED FILE-----
+        YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuclJTYVhER1Q2UUh5Qzl6
+        RWtkUGp5QjJwbnI1L3RYYXlzWjRQVEx2MlNzClNuOXl2VUhHcDVDVzRyZE9la1o3
+        alFqWXF1UUU0Q0sxMzkwelRvWndzVlkKLS0tIG5KY1hKWnZ6TkNLN1VHSGEvN2c4
+        M1F2Q01ldjZBaHNXaTh0cTRNU2RrWjQKEbGIRnnabeu08gLr8DfAAtbDVGHOFN1A
+        dzBAGPYN3Ws4zILpVEVXg8xMgf2phWpTRd0/OHPLes/iXVPogVHBpQ==
+        -----END AGE ENCRYPTED FILE-----
+  lastmodified: "2024-11-25T18:47:12Z"
+  mac: ENC[AES256_GCM,data:eo6m1DNow8fEwutlB2zzVV8mYxKLsqHZgdWyE0uMVT3k5X9NsqbZkdZYhA2iSxmLBq3GVATRBbJS25bibX8gGvHFLdtc3zNHJ3TNDhbgjwR68MTTIu3b+61rGDyMEvagySIboJ9mOJOm8U5SzrkB0ln50RvUonyZgftcDW1r/Hc=,iv:DbxQgtR0dX2922LmFoqaeDVnztQxbY3h8xIrN4JHjUw=,tag:vYa5SzVgRSfF4N/c8M8CAg==,type:str]
+  pgp: []
+  encrypted_regex: ^(data|stringData)$
+  version: 3.9.1

kubernetes/arc1/apps/dev/gha-runner-scale-set-controller/ks.yaml

@@ -0,0 +1,26 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app gha-runner-scale-set-controller
+  namespace: flux-system
+spec:
+  targetNamespace: dev
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  dependsOn:
+    - name: traefik
+  path: ./kubernetes/arc1/apps/dev/gha-runner-scale-set-controller/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: k8s-gitops
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+  postBuild:
+    substitute:
+      APP: *app

kubernetes/arc1/apps/dev/gha-runner-scale-set/app/helmrelease.yaml

@@ -0,0 +1,53 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: &app gha-runner-scale-set
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: gha-runner-scale-set
+      version: 0.9.3
+      sourceRef:
+        kind: HelmRepository
+        name: actions-runner-controller
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  valuesFrom:
+    - targetPath: githubConfigSecret.github_app_id
+      kind: Secret
+      name: actions-runner-controller-secret
+      valuesKey: github_app_id
+    - targetPath: githubConfigSecret.github_app_installation_id
+      kind: Secret
+      name: actions-runner-controller-secret
+      valuesKey: github_app_installation_id
+    - targetPath: githubConfigSecret.github_app_private_key
+      kind: Secret
+      name: actions-runner-controller-secret
+      valuesKey: github_app_private_key
+  values:
+    nameOverride: *app
+    runnerScaleSetName: *app
+    githubConfigUrl: https://github.com/ergho/homelab-ops
+    minRunners: 1
+    maxRunners: 3
+    containerMode:
+      type: dind
+    template:
+      spec:
+        containers:
+          - name: runner
+            image: ghcr.io/rarecompute/actions-runner:2.320.0@sha256:a357b0db3cb504da87cbbd46b304617fbf4a579e2d0f850ea4f177042f0a4786
+            command: ["/home/runner/run.sh"]
+    controllerServiceAccount:
+      name: gha-runner-scale-set-controller
+      namespace: actions-runner-system

kubernetes/arc1/apps/dev/gha-runner-scale-set/app/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml

kubernetes/arc1/apps/dev/gha-runner-scale-set/ks.yaml

@@ -0,0 +1,26 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app gha-runner-scale-set
+  namespace: flux-system
+spec:
+  targetNamespace: dev
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  dependsOn:
+    - name: traefik
+  path: ./kubernetes/arc1/apps/dev/gha-runner-scale-set/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: k8s-gitops
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+  postBuild:
+    substitute:
+      APP: *app

kubernetes/arc1/apps/dev/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./namespace.yaml
+  - ./gha-runner-scale-set-controller/ks.yaml
+  - ./gha-runner-scale-set/ks.yaml

kubernetes/arc1/apps/dev/namespace.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dev
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled

kubernetes/arc1/apps/machine-learning/boltz/README.md

@@ -0,0 +1,5 @@
+<!-- Source: https://github.com/jwohlwend/boltz-->
+
+We have containerized [Boltz-1](https://github.com/jwohlwend/boltz) for testing and development purposes
+
+Boltz-1 is an open-source model which predicts the 3D structure of proteins, RNA, DNA and small molecules; it handles modified residues, covalent ligands and glycans, as well as condition the generation on pocket residues.

kubernetes/arc1/apps/machine-learning/boltz/app/helmrelease.yaml

@@ -0,0 +1,92 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json
+# TODO: Finish this
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: &app boltz
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: app-template
+      version: 3.5.1
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  values:
+    controllers:
+      boltz:
+        type: deployment
+        annotations:
+          reloader.stakater.com/auto: "true"
+        pod:
+          runtimeClassName: nvidia
+          terminationGracePeriodSeconds: 1
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                  - matchExpressions:
+                      - key: nvidia.com/gpu.present
+                        operator: In
+                        values:
+                          - "true"
+        containers:
+          app:
+            image:
+              repository: ghcr.io/rarecompute/boltz-docker-ci
+              # Base image
+              #tag: main@sha256:e2258b1a2ade6c5e92ee0b5e314fc8a073ca15d06bc4b6b6f32b0699ff958b96
+
+              # CUDA 12.1
+              #tag: cuda12-1@sha256:c4da98989ade648a5158b01acc0d86648410ba9e73bfc281564aa8e000bd67ed
+
+              # CUDA 12.4
+              tag: cuda12-4@sha256:342069ec496df0027c355a5f7418732b8b0e86931e7ed2dfc3bc679b73b43e0b
+            command: ["tail", "-f", "/dev/null"]
+            env:
+              TZ: ${TIMEZONE}
+              NVIDIA_VISIBLE_DEVICES: all
+              NVIDIA_DRIVER_CAPABILITIES: all
+            securityContext:
+              capabilities.drop: ["ALL"]
+            resources:
+              requests:
+                cpu: 200m
+                memory: 8Gi
+              limits:
+                cpu: 16
+                memory: 48Gi
+                nvidia.com/gpu: 4
+    service:
+      app:
+        controller: *app
+        annotations:
+        ports:
+          http:
+            port: &port 80
+    persistence:
+      #      app:
+      #        storageClass: local-nvme
+      #        accessMode: ReadWriteOnce
+      #        size: 1Gi
+      #        globalMounts:
+      #          - path: /app
+      workspace:
+        existingClaim: qlora-workspace
+        globalMounts:
+          - path: /workspace
+      tmp:
+        type: emptyDir
+        globalMounts:
+          - path: /tmp

kubernetes/arc1/apps/machine-learning/boltz/app/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml

kubernetes/arc1/apps/machine-learning/boltz/ks.yaml

@@ -0,0 +1,24 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app boltz
+  namespace: flux-system
+spec:
+  targetNamespace: machine-learning
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/arc1/apps/machine-learning/boltz/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: k8s-gitops
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+  postBuild:
+    substitute:
+      APP: *app

kubernetes/arc1/apps/machine-learning/colabfold/app/helmrelease.yaml

@@ -26,7 +26,7 @@ spec:
       retries: 3
   values:
     controllers:
-      ollama:
+      colabfold:
         type: deployment
         annotations:
           reloader.stakater.com/auto: "true"
@@ -56,11 +56,11 @@ spec:
               requests:
                 cpu: 200m
                 memory: 4Gi
-                gpu.intel.com/i915: "4"
+                nvidia.com/gpu: 2
               limits:
                 cpu: 32000m
                 memory: 64Gi
-                gpu.intel.com/i915: "4"
+                nvidia.com/gpu: 4
     service:
       app:
         controller: *app
@@ -88,6 +88,7 @@ spec:
           - secretName: colabfold-tls
             hosts: [*host]
     persistence:
+      # TODO: Replace with existing PVC
       data:
         storageClass: local-nvme
         accessMode: ReadWriteMany

kubernetes/arc1/apps/machine-learning/kustomization.yaml

@@ -6,4 +6,5 @@ resources:
   - ./namespace.yaml
   - ./qlora/ks.yaml
   - ./mmseqs2/ks.yaml
+  - ./boltz/ks.yaml
   #- ./jupyterhub/ks.yaml

kubernetes/arc1/apps/machine-learning/mmseqs2/README.md

@@ -0,0 +1,3 @@
+<!-- Source: https://github.com/soedinglab/MMseqs2 -->
+
+MMseqs2 (Many-against-Many sequence searching) is a software suite to search and cluster huge protein and nucleotide sequence sets. MMseqs2 is free and open source software implemented in C++ for Linux, MacOS, and (as beta version, via cygwin) Windows. The software is designed to run on multiple cores and servers and exhibits very good scalability. MMseqs2 can run 10000 times faster than BLAST. At 100 times its speed it achieves almost the same sensitivity. It can perform profile searches with the same sensitivity as PSI-BLAST at over 400 times its speed.

kubernetes/arc1/apps/machine-learning/mmseqs2/app/helmrelease.yaml

@@ -60,7 +60,7 @@ spec:
             resources:
               requests:
                 cpu: 200m
-                memory: 8Gi
+                memory: 4Gi
               limits:
                 cpu: 16
                 memory: 32Gi

kubernetes/arc1/apps/machine-learning/qlora/README.md

@@ -0,0 +1,7 @@
+We have containerized [QLoRA](https://arxiv.org/abs/2305.14314) for testing and development purposes
+
+From [artidoro/qlora](https://github.com/artidoro/qlora)
+
+> QLoRA uses bitsandbytes for quantization and is integrated with Hugging Face's PEFT and transformers libraries. QLoRA was developed by members of the University of Washington's UW NLP group.
+>
+> We present QLoRA, an efficient finetuning approach that reduces memory usage enough to finetune a 65B parameter model on a single 48GB GPU while preserving full 16-bit finetuning task performance. QLoRA backpropagates gradients through a frozen, 4-bit quantized pretrained language model into Low Rank Adapters (LoRA). Our best model family, which we name Guanaco, outperforms all previous openly released models on the Vicuna benchmark, reaching 99.3% of the performance level of ChatGPT while only requiring 24 hours of finetuning on a single GPU. QLoRA introduces a number of innovations to save memory without sacrificing performance: (a) 4-bit NormalFloat (NF4), a new data type that is information theoretically optimal for normally distributed weights (b) Double Quantization to reduce the average memory footprint by quantizing the quantization constants, and (c) Paged Optimizers to manage memory spikes. We use QLoRA to finetune more than 1,000 models, providing a detailed analysis of instruction following and chatbot performance across 8 instruction datasets, multiple model types (LLaMA, T5), and model scales that would be infeasible to run with regular finetuning (e.g. 33B and 65B parameter models). Our results show that QLoRA finetuning on a small high-quality dataset leads to state-of-the-art results, even when using smaller models than the previous SoTA. We provide a detailed analysis of chatbot performance based on both human and GPT-4 evaluations showing that GPT-4 evaluations are a cheap and reasonable alternative to human evaluation. Furthermore, we find that current chatbot benchmarks are not trustworthy to accurately evaluate the performance levels of chatbots. We release all of our models and code, including CUDA kernels for 4-bit training.