From c2d225309525bf239b063e1b5c974aa8a6931acc Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Sat, 20 Jan 2024 08:28:31 -0500 Subject: [PATCH] chore: update taskfiles and use requires Signed-off-by: Devin Buhl --- .taskfiles/Ansible/Taskfile.yaml | 10 ++--- .taskfiles/ExternalSecrets/Taskfile.yaml | 10 ++--- .taskfiles/Flux/Taskfile.yaml | 25 ++++++----- .taskfiles/Kubernetes/Taskfile.yaml | 6 +-- .taskfiles/VolSync/Taskfile.yaml | 56 ++++++++++++++---------- 5 files changed, 59 insertions(+), 48 deletions(-) diff --git a/.taskfiles/Ansible/Taskfile.yaml b/.taskfiles/Ansible/Taskfile.yaml index 05482a4fd9be4..0f8166c9dfc85 100644 --- a/.taskfiles/Ansible/Taskfile.yaml +++ b/.taskfiles/Ansible/Taskfile.yaml @@ -32,12 +32,12 @@ tasks: .venv/bin/ansible-playbook \ --inventory {{.ANSIBLE_DIR}}/{{.cluster}}/inventory/hosts.yaml \ {{.ANSIBLE_DIR}}/{{.cluster}}/playbooks/{{.playbook}}.yaml {{.CLI_ARGS}} + requires: + vars: ["cluster", "playbook"] preconditions: - - { msg: "Argument (cluster) is required", sh: "test -n {{.cluster}}" } - - { msg: "Argument (playbook) is required", sh: "test -n {{.playbook}}" } - - { msg: "Venv not found", sh: "test -d {{.ROOT_DIR}}/.venv" } - - { msg: "Inventory not found", sh: "test -f {{.ANSIBLE_DIR}}/{{.cluster}}/inventory/hosts.yaml" } - - { msg: "Playbook not found", sh: "test -f {{.ANSIBLE_DIR}}/{{.cluster}}/playbooks/{{.playbook}}.yaml" } + - { msg: "Venv not found", sh: "test -d {{.ROOT_DIR}}/.venv" } + - { msg: "Inventory not found", sh: "test -f {{.ANSIBLE_DIR}}/{{.cluster}}/inventory/hosts.yaml" } + - { msg: "Playbook not found", sh: "test -f {{.ANSIBLE_DIR}}/{{.cluster}}/playbooks/{{.playbook}}.yaml" } .venv: internal: true diff --git a/.taskfiles/ExternalSecrets/Taskfile.yaml b/.taskfiles/ExternalSecrets/Taskfile.yaml index 67a971d131ffc..85237159a0fed 100644 --- a/.taskfiles/ExternalSecrets/Taskfile.yaml +++ b/.taskfiles/ExternalSecrets/Taskfile.yaml @@ -14,12 +14,12 @@ tasks: cmd: kubectl --context {{.cluster}} -n {{.ns}} annotate externalsecret {{.secret}} force-sync=$(date +%s) --overwrite env: KUBECONFIG: "{{.KUBERNETES_DIR}}/{{.cluster}}/kubeconfig" + requires: + vars: ["cluster", "secret"] vars: ns: '{{.ns | default "default"}}' - secret: '{{ or .secret (fail "Argument (secret) is required") }}' preconditions: - - { msg: "ExternalSecret not found", sh: "kubectl --context {{.cluster}} -n {{.ns}} get externalsecret {{.secret}}" } - - { msg: "Argument (cluster) is required", sh: "test -n {{.cluster}}" } + - { msg: "ExternalSecret not found", sh: "kubectl --context {{.cluster}} -n {{.ns}} get externalsecret {{.secret}}" } sync-all: desc: Sync all ExternalSecrets for a cluster @@ -35,8 +35,8 @@ tasks: secret: '{{$a := split "|" .ITEM}}{{$a._1}}' env: KUBECONFIG: "{{.KUBERNETES_DIR}}/{{.cluster}}/kubeconfig" + requires: + vars: ["cluster"] vars: secrets: sh: kubectl --context {{.cluster}} get externalsecret --all-namespaces --no-headers -A | awk '{print $1 "|" $2}' - preconditions: - - { msg: "Argument (cluster) is required", sh: "test -n {{.cluster}}" } diff --git a/.taskfiles/Flux/Taskfile.yaml b/.taskfiles/Flux/Taskfile.yaml index 969946c82b4ae..c28aaa1147bcc 100644 --- a/.taskfiles/Flux/Taskfile.yaml +++ b/.taskfiles/Flux/Taskfile.yaml @@ -28,17 +28,18 @@ tasks: - kubectl --context {{.cluster}} apply --server-side --kustomize {{.KUBERNETES_DIR}}/{{.cluster}}/flux/vars - kubectl --context {{.cluster}} apply --server-side --kustomize {{.KUBERNETES_DIR}}/{{.cluster}}/flux/config - defer: sops --encrypt --in-place {{.KUBERNETES_DIR}}/{{.cluster}}/flux/vars/cluster-secrets.secret.sops.env + requires: + vars: ["cluster"] preconditions: - - { msg: "Argument (cluster) is required", sh: "test -n {{.cluster}}" } - - { msg: "Flux appears already installed", sh: "exit $(( ! $(kubectl --context {{.cluster}} get namespace flux-system) ))" } - - { msg: "Age private key not found", sh: "test -f {{.ROOT_DIR}}/age.key" } - - { msg: "Age secret not found", sh: "test -f {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/flux/age-key.secret.sops.yaml" } + - { msg: "Flux appears already installed", sh: "kubectl --context {{.cluster}} get namespace flux-system &>/dev/null && exit 1 || exit 0" } + - { msg: "Age private key not found", sh: "test -f {{.ROOT_DIR}}/age.key" } + - { msg: "Age secret not found", sh: "test -f {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/flux/age-key.secret.sops.yaml" } - { msg: "Github deploy secret not found", sh: "test -f {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/flux/github-deploy-key.secret.sops.yaml" } - - { msg: "Cluster settings not found", sh: "test -f {{.KUBERNETES_DIR}}/{{.cluster}}/flux/vars/cluster-settings.env" } - - { msg: "Cluster secrets not found", sh: "test -f {{.KUBERNETES_DIR}}/{{.cluster}}/flux/vars/cluster-secrets.secret.sops.env" } - - { msg: "Unable to decrypt sops secret", sh: "sops --decrypt {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/flux/age-key.secret.sops.yaml" } - - { msg: "Unable to decrypt sops secret", sh: "sops --decrypt {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/flux/github-deploy-key.secret.sops.yaml" } - - { msg: "Unable to decrypt sops secret", sh: "sops --decrypt {{.KUBERNETES_DIR}}/{{.cluster}}/flux/vars/cluster-secrets.secret.sops.env" } + - { msg: "Cluster settings not found", sh: "test -f {{.KUBERNETES_DIR}}/{{.cluster}}/flux/vars/cluster-settings.env" } + - { msg: "Cluster secrets not found", sh: "test -f {{.KUBERNETES_DIR}}/{{.cluster}}/flux/vars/cluster-secrets.secret.sops.env" } + - { msg: "Unable to decrypt sops secret", sh: "sops --decrypt {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/flux/age-key.secret.sops.yaml" } + - { msg: "Unable to decrypt sops secret", sh: "sops --decrypt {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/flux/github-deploy-key.secret.sops.yaml" } + - { msg: "Unable to decrypt sops secret", sh: "sops --decrypt {{.KUBERNETES_DIR}}/{{.cluster}}/flux/vars/cluster-secrets.secret.sops.env" } apply: desc: Apply a Flux Kustomization resource for a cluster @@ -56,11 +57,11 @@ tasks: | \ kubectl --context {{.cluster}} apply --server-side \ --field-manager=kustomize-controller -f - + requires: + vars: ["cluster", "path"] vars: - path: '{{ or .path (fail "Argument (path) is required") }}' ns: '{{.ns | default "flux-system"}}' ks: sh: flux --context {{.cluster}} --namespace {{.ns}} get kustomizations $(basename {{.path}}) 2>&1 preconditions: - - { msg: "Argument (cluster) is required", sh: "test -n {{.cluster}}" } - - { msg: "Kustomization file not found", sh: "test -f {{.KUBERNETES_DIR}}/{{.cluster}}/apps/{{.path}}/ks.yaml" } + - { msg: "Kustomization file not found", sh: "test -f {{.KUBERNETES_DIR}}/{{.cluster}}/apps/{{.path}}/ks.yaml" } diff --git a/.taskfiles/Kubernetes/Taskfile.yaml b/.taskfiles/Kubernetes/Taskfile.yaml index 72e73c778d7a1..9377c1986a719 100644 --- a/.taskfiles/Kubernetes/Taskfile.yaml +++ b/.taskfiles/Kubernetes/Taskfile.yaml @@ -44,9 +44,9 @@ tasks: "restartPolicy": "Never" } }' + requires: + vars: ["cluster", "claim"] vars: ns: '{{.ns | default "default"}}' - claim: '{{ or .claim (fail "Argument (claim) is required") }}' preconditions: - - { msg: "Argument (cluster) is required", sh: "test -n {{.cluster}}" } - - { msg: "PVC not found", sh: "kubectl --context {{.cluster}} -n {{.ns}} get persistentvolumeclaim {{.claim}}" } + - { msg: "PVC not found", sh: "kubectl --context {{.cluster}} -n {{.ns}} get persistentvolumeclaim {{.claim}}" } diff --git a/.taskfiles/VolSync/Taskfile.yaml b/.taskfiles/VolSync/Taskfile.yaml index b2c3d1e95ac20..4e32d2ab701e2 100644 --- a/.taskfiles/VolSync/Taskfile.yaml +++ b/.taskfiles/VolSync/Taskfile.yaml @@ -38,13 +38,13 @@ tasks: - kubectl --context {{.cluster}} -n {{.ns}} logs job/list-{{.app}}-{{.ts}} --container list - kubectl --context {{.cluster}} -n {{.ns}} delete job list-{{.app}}-{{.ts}} env: *env + requires: + vars: ["cluster", "app"] vars: ns: '{{.ns | default "default"}}' - app: '{{ or .app (fail "Argument (app) is required") }}' preconditions: - - { msg: "Argument (cluster) is required", sh: "test -n {{.cluster}}" } - - { msg: "Wait script not found", sh: "test -f {{.scriptsDir}}/wait.sh" } - - { msg: "List template not found", sh: "test -f {{.templatesDir}}/list.tmpl.yaml" } + - { msg: "Wait script not found", sh: "test -f {{.scriptsDir}}/wait.sh" } + - { msg: "List template not found", sh: "test -f {{.templatesDir}}/list.tmpl.yaml" } silent: true unlock: @@ -62,13 +62,13 @@ tasks: - kubectl --context {{.cluster}} -n {{.ns}} logs job/{{.app}}-unlock-snapshots-{{.ts}} --container unlock-r2 - kubectl --context {{.cluster}} -n {{.ns}} delete job unlock-{{.app}}-{{.ts}} env: *env + requires: + vars: ["cluster", "app"] vars: ns: '{{.ns | default "default"}}' - app: '{{ or .app (fail "Argument (app) is required") }}' preconditions: - - { msg: "Argument (cluster) is required", sh: "test -n {{.cluster}}" } - - { msg: "Wait script not found", sh: "test -f {{.scriptsDir}}/wait.sh" } - - { msg: "Unlock template not found", sh: "test -f {{.templatesDir}}/unlock.tmpl.yaml" } + - { msg: "Wait script not found", sh: "test -f {{.scriptsDir}}/wait.sh" } + - { msg: "Unlock template not found", sh: "test -f {{.templatesDir}}/unlock.tmpl.yaml" } silent: true # To run backup jobs in parallel for all replicationsources: @@ -89,17 +89,16 @@ tasks: - bash {{.scriptsDir}}/wait.sh volsync-src-{{.app}} {{.ns}} {{.cluster}} - kubectl --context {{.cluster}} -n {{.ns}} wait job/volsync-src-{{.app}} --for condition=complete --timeout=120m env: *env + requires: + vars: ["cluster", "app"] vars: - cluster: '{{ or .cluster (fail "Argument (cluster) is required") }}' ns: '{{.ns | default "default"}}' - app: '{{ or .app (fail "Argument (app) is required") }}' controller: sh: true && {{.scriptsDir}}/controller.sh {{.app}} {{.ns}} {{.cluster}} preconditions: - - { msg: "Argument (cluster) is required", sh: "test -n {{.cluster}}" } - - { msg: "Controller script not found", sh: "test -f {{.scriptsDir}}/controller.sh" } - - { msg: "Wait script not found", sh: "test -f {{.scriptsDir}}/wait.sh" } - - { msg: "RepositorySource not found", sh: "kubectl --context {{.cluster}} -n {{.ns}} get replicationsources {{.app}}" } + - { msg: "Controller script not found", sh: "test -f {{.scriptsDir}}/controller.sh" } + - { msg: "Wait script not found", sh: "test -f {{.scriptsDir}}/wait.sh" } + - { msg: "RepositorySource not found", sh: "kubectl --context {{.cluster}} -n {{.ns}} get replicationsources {{.app}}" } # To run restore jobs in parallel for all replicationdestinations: # - kubectl get replicationsources --all-namespaces --no-headers | awk '{print $2, $1}' | xargs --max-procs=4 -l bash -c 'task volsync:restore app=$0 ns=$1' @@ -121,9 +120,10 @@ tasks: - task: .resume vars: *env env: *env + requires: + vars: ["cluster", "app"] vars: ns: '{{.ns | default "default"}}' - app: '{{ or .app (fail "Argument (app) is required") }}' previous: '{{.previous | default 2}}' controller: sh: "{{.scriptsDir}}/controller.sh {{.app}} {{.ns}}" @@ -134,11 +134,10 @@ tasks: pgid: sh: kubectl --context {{.cluster}} -n {{.ns}} get replicationsources/{{.app}} -o jsonpath="{.spec.restic.moverSecurityContext.runAsGroup}" preconditions: - - { msg: "Argument (cluster) is required", sh: "test -n {{.cluster}}" } - - { msg: "Controller script not found", sh: "test -f {{.scriptsDir}}/controller.sh" } - - { msg: "Wait script not found", sh: "test -f {{.scriptsDir}}/wait.sh" } + - { msg: "Controller script not found", sh: "test -f {{.scriptsDir}}/controller.sh" } + - { msg: "Wait script not found", sh: "test -f {{.scriptsDir}}/wait.sh" } - { msg: "ReplicationDestination script not found", sh: "test -f {{.templatesDir}}/replicationdestination.tmpl.yaml" } - - { msg: "Wipe template not found", sh: "test -f {{.templatesDir}}/wipe.tmpl.yaml" } + - { msg: "Wipe template not found", sh: "test -f {{.templatesDir}}/wipe.tmpl.yaml" } cleanup: desc: Delete volume populator PVCs in all namespaces @@ -146,11 +145,22 @@ tasks: Args: cluster: Cluster to run command against (required) cmds: - - kubectl --context {{.cluster}} get pvc --all-namespaces --no-headers | grep "dst-dest" | awk '{print $2, $1}' | xargs -l bash -c 'kubectl --context {{.cluster}} delete pvc -n $1 $0' - - kubectl --context {{.cluster}} get pvc --all-namespaces --no-headers | grep "dst-cache" | awk '{print $2, $1}' | xargs -l bash -c 'kubectl --context {{.cluster}} delete pvc -n $1 $0' + - for: { var: dest } + cmd: | + {{- $items := (split "/" .ITEM) }} + kubectl --context {{.cluster}} delete pvc -n {{ $items._0 }} {{ $items._1 }} + - for: { var: cache } + cmd: | + {{- $items := (split "/" .ITEM) }} + kubectl --context {{.cluster}} delete pvc -n {{ $items._0 }} {{ $items._1 }} env: *env - preconditions: - - { msg: "Argument (cluster) is required", sh: "test -n {{.cluster}}" } + requires: + vars: ["cluster"] + vars: + dest: + sh: kubectl --context {{.cluster}} get pvc --all-namespaces --no-headers | grep "dst-dest" | awk '{print $1 "/" $2}' + cache: + sh: kubectl --context {{.cluster}} get pvc --all-namespaces --no-headers | grep "dst-cache" | awk '{print $1 "/" $2}' # Suspend the Flux ks and hr .suspend: