feat(taskfiles): add talos rollout upgrade task

Signed-off-by: Devin Buhl <[email protected]>
onedr0p · Feb 19, 2024 · 62eb264 · 62eb264
1 parent b6c26f8
commit 62eb264
Showing 1 changed file with 60 additions and 22 deletions.
diff --git a/.taskfiles/Talos/Taskfile.yaml b/.taskfiles/Talos/Taskfile.yaml
@@ -49,36 +49,16 @@ tasks:
   bootstrap-apps:
     desc: Bootstrap core apps needed for Talos
     cmds:
-      - until kubectl --context {{.cluster}} wait --for=condition=Ready=False nodes --all --timeout=600s; do sleep 10; done
+      - until kubectl --context {{.cluster}} wait --for=condition=Ready=False nodes --all --timeout=10m; do sleep 10; done
       - helmfile --quiet --kube-context {{.cluster}} --file {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/apps/helmfile.yaml apply --skip-diff-on-install --suppress-diff
-      - until kubectl --context {{.cluster}} wait --for=condition=Ready nodes --all --timeout=600s; do sleep 10; done
+      - until kubectl --context {{.cluster}} wait --for=condition=Ready nodes --all --timeout=10m; do sleep 10; done
     requires:
       vars: ["cluster"]
     preconditions:
       - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig
       - talosctl --context {{.cluster}} config info >/dev/null 2>&1
       - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/apps/helmfile.yaml
 
-  upgrade-talos:
-    desc: Upgrade Talos on a node
-    cmd: talosctl --context {{.cluster}} --nodes {{.node}} upgrade --image {{.image}} --preserve=true
-    requires:
-      vars: ["node", "image"]
-    preconditions:
-      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig
-      - talosctl --context {{.cluster}} config info >/dev/null 2>&1
-      - talosctl --context {{.cluster}} --nodes {{.node}} get machineconfig >/dev/null 2>&1
-
-  upgrade-k8s:
-    desc: Upgrade k8s on a node
-    cmd: talosctl --context {{.cluster}} --nodes {{.node}} upgrade-k8s --to {{.to}}
-    requires:
-      vars: ["node", "to"]
-    preconditions:
-      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig
-      - talosctl --context {{.cluster}} config info >/dev/null 2>&1
-      - talosctl --context {{.cluster}} --nodes {{.node}} get machineconfig >/dev/null 2>&1
-
   fetch-kubeconfig:
     desc: Fetch kubeconfig from Talos controllers
     cmd: |
@@ -140,3 +120,61 @@ tasks:
       KUBERNETES_VERSION: "{{.KUBERNETES_VERSION}}"
     requires:
       vars: ["cluster"]
+
+  upgrade-rollout:
+    desc: Rollout Talos upgrade on all nodes
+    cmds:
+      - flux --context {{.cluster}} suspend kustomization --all
+      - kubectl cnpg --context {{.cluster}} maintenance set --reusePVC --all-namespaces
+      - for: { var: nodes }
+        task: upgrade
+        vars:
+          node: "{{.ITEM}}"
+          rollout: "true"
+      - kubectl cnpg --context {{.cluster}} maintenance unset --reusePVC --all-namespaces
+      - flux --context {{.cluster}} resume kustomization --all
+      - task: :kubernetes:delete-failed-pods
+        vars:
+          cluster: "{{.cluster}}"
+    vars:
+      nodes:
+        sh: talosctl --context {{.cluster}} config info --output json | jq --join-output '[.nodes[]] | join(" ")'
+    requires:
+      vars: ["cluster"]
+    preconditions:
+      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig
+      - talosctl --context {{.cluster}} config info >/dev/null 2>&1
+
+  upgrade:
+    desc: Upgrade Talos on a node
+    cmds:
+      - kubectl --context {{.cluster}} wait --for=condition=Complete --timeout=10m jobs --all --all-namespaces
+      - '{{if not (eq .rollout "true")}}flux --context {{.cluster}} suspend kustomization --all{{end}}'
+      - '{{if not (eq .rollout "true")}}kubectl cnpg --context {{.cluster}} maintenance set --reusePVC --all-namespaces{{end}}'
+      - kubectl --context {{.cluster}} drain {{.nodename}} --ignore-daemonsets --delete-emptydir-data --force
+      - talosctl --context {{.cluster}} --nodes {{.node}} upgrade --image=factory.talos.dev/installer/{{.TALOS_SCHEMATIC_ID}}:{{.TALOS_VERSION}} --preserve=true
+      - talosctl --context {{.cluster}} --nodes {{.node}} health --wait-timeout=10m --server=false
+      - '{{if not (eq .rollout "true")}}kubectl --context {{.cluster}} wait --timeout=10m --for=jsonpath=.status.ceph.health=HEALTH_OK cephcluster --all --all-namespaces{{end}}'
+      - '{{if not (eq .rollout "true")}}flux --context {{.cluster}} resume kustomization --all{{end}}'
+    vars:
+      nodename:
+        sh: kubectl --context {{.cluster}} get nodes -o jsonpath='{range .items[?(.status.addresses[0].address=="{{.node}}")]}{.metadata.name}'
+    requires:
+      vars: ["cluster", "node"]
+    preconditions:
+      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig
+      - talosctl --context {{.cluster}} config info >/dev/null 2>&1
+      - talosctl --context {{.cluster}} --nodes {{.node}} get machineconfig >/dev/null 2>&1
+
+  upgrade-k8s:
+    desc: Upgrade the clusters k8s version
+    cmd: talosctl --context {{.cluster}} --nodes {{.controller}} upgrade-k8s --to {{.KUBERNETES_VERSION}}
+    vars:
+      controller:
+        sh: talosctl --context {{.cluster}} config info --output json | jq --raw-output '.endpoints[0]'
+    requires:
+      vars: ["cluster"]
+    preconditions:
+      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/talosconfig
+      - talosctl --context {{.cluster}} config info >/dev/null 2>&1
+      - talosctl --context {{.cluster}} --nodes {{.node}} get machineconfig >/dev/null 2>&1