From 49dc5f4a495a92ec777fcc5b2fd5f6385d3048c8 Mon Sep 17 00:00:00 2001 From: haasad Date: Thu, 23 May 2024 15:10:23 +0200 Subject: [PATCH] Add how-to for cloudscale API token rotation --- .../how-tos/cloudscale/rotate-api-tokens.adoc | 210 ++++++++++++++++++ docs/modules/ROOT/partials/nav.adoc | 2 + 2 files changed, 212 insertions(+) create mode 100644 docs/modules/ROOT/pages/how-tos/cloudscale/rotate-api-tokens.adoc diff --git a/docs/modules/ROOT/pages/how-tos/cloudscale/rotate-api-tokens.adoc b/docs/modules/ROOT/pages/how-tos/cloudscale/rotate-api-tokens.adoc new file mode 100644 index 00000000..ad8b7664 --- /dev/null +++ b/docs/modules/ROOT/pages/how-tos/cloudscale/rotate-api-tokens.adoc @@ -0,0 +1,210 @@ += Rotate cluster API Tokens + +[abstract] +-- +Steps to rotate the API tokens used by a cloudscale.ch OpenShift4 cluster and its surrounding tooling. +-- + +== Prerequisites + +* https://kubernetes.io/docs/tasks/tools/#kubectl[`kubectl`] +* `yq` https://mikefarah.gitbook.io/yq[yq YAML processor] (version 4 or higher - use the go version by mikefarah, not the jq wrapper by kislyuk) +* `vault` https://www.vaultproject.io/docs/commands[Vault CLI] + +== Rotate the main cluster API token + +. Select the cluster whose token is being rotated. ++ +[source,bash] +---- +export CLUSTER_ID= +export TENANT_ID=$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${CLUSTER_ID} | jq -r .tenant) +---- + +. Identify the API token that should be replaced. +.. Got to https://control.cloudscale.ch/service//api-token. +.. The token should be named ``. + +. Create a new API token with read/write permissions and name it again ``. The token names do not need to be unique. ++ +[source,bash] +---- +export CLOUDSCALE_API_TOKEN= +---- + +. Update the token in vault. ++ +include::partial$connect-to-vault.adoc[] ++ +[source,bash] +---- +vault kv patch clusters/kv/${TENANT_ID}/${CLUSTER_ID}/cloudscale \ + token=${CLOUDSCALE_API_TOKEN} +---- + +. Connect to ArgoCD and "hard refresh" all apps. + +. Verify that the `cloudscale` secret has been updated. ++ +[source,bash] +---- +kubectl --as cluster-admin -n syn-csi-cloudscale \ + get secrets cloudscale -oyaml |\ + yq '.data.access-token' | base64 -d +---- + +. Restart the csi-cloudscale-controller. ++ +[source,bash] +---- +kubectl --as cluster-admin -n syn-csi-cloudscale rollout restart \ + sts csi-cloudscale-controller +---- + +. Verify that the new token is used by the csi-driver. +.. Create a namspace with a PVC and a pod that mounts the volume. ++ +[source,bash] +---- +cat < Pipelines -> Run pipeline). +.. Verify in the cloudscale UI that the new token has been used. + +. Delete the old API token. ++ +[WARNING] +==== +There are two tokens with the same name. Make sure to delete the one that hasn't been used since the rotation. +==== + + +== Rotate the floaty API token + +. Select the cluster whose token is being rotated. ++ +[source,bash] +---- +export CLUSTER_ID= +export TENANT_ID=$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${CLUSTER_ID} | jq -r .tenant) +---- + +. Identify the API token that should be replaced. +.. Got to https://control.cloudscale.ch/service//api-token. +.. The token should be named `_floaty`. + +. Create a new API token with read/write permissions and name it again `_floaty`. The token names do not need to be unique. ++ +[source,bash] +---- +export FLOATY_TOKEN= +---- + +. Update the token in vault. ++ +include::partial$connect-to-vault.adoc[] ++ +[source,bash] +---- +vault kv put clusters/kv/${TENANT_ID}/${CLUSTER_ID}/floaty \ + iam_secret=${FLOATY_TOKEN} +---- + +. Update the `CLOUDSCALE_FLOATY_SECRET` CI/CD variable in the the cluster-catalog repository. +.. Go to the cluster-catalog repo's CI/CD settings. ++ +[source,bash] +---- +url="https://$(commodore catalog list -oyaml |\ + yq '.[] | select(.id == strenv(CLUSTER_ID)) | .gitRepo.url' |\ + sed -E 's/.+@(.+).git/\1/')/-/settings/ci_cd" +xdg-open $url || open $url || echo $url +---- +.. Edit and update the value of the `CLOUDSCALE_FLOATY_SECRET` variable with the new API token. +.. Trigger a pipeline run (Build -> Pipelines -> Run pipeline). +.. Check the terraform plan output for any unrelated changes, expected output: ++ +[source] +---- +Plan: 2 to add, 0 to change, 0 to destroy. +---- +.. Run the `apply` stage of the pipeline. +.. Open the linked merge request in the terraform output of the `apply` job. + +. Review and merge the MR created by terrafrom in the APPUiO hieradata. +.. Wait for the `mco_git` deploy pipeline to finish before continuing. + +. Run puppet on both LBs. ++ +[source,bash] +---- +ssh enc.appuio.lbaas.$CLUSTER_ID.lb.0 sudo puppetctl run +ssh enc.appuio.lbaas.$CLUSTER_ID.lb.1 sudo puppetctl run +---- +.. Verify that the tokens are updated in the puppet output. + +. Verify in the cloudscale UI that the new token is being used and usage of the old token has stopped. + +. Delete the old API token. ++ +[WARNING] +==== +There are two tokens with the same name. Make sure to delete the one that hasn't been used since the rotation. +==== diff --git a/docs/modules/ROOT/partials/nav.adoc b/docs/modules/ROOT/partials/nav.adoc index f9a15ec2..091bfefa 100644 --- a/docs/modules/ROOT/partials/nav.adoc +++ b/docs/modules/ROOT/partials/nav.adoc @@ -41,6 +41,7 @@ *** xref:oc4:ROOT:how-tos/cloudscale/enable-loadbalancer-service.adoc[Enable LoadBalancer Services] *** xref:oc4:ROOT:how-tos/cloudscale/recover-etcd.adoc[Restore etcd] *** xref:oc4:ROOT:how-tos/cloudscale/decommission.adoc[Decommissioning] +*** xref:oc4:ROOT:how-tos/cloudscale/rotate-api-tokens.adoc[Rotate API Tokens] ** Exoscale *** xref:oc4:ROOT:references/exoscale/architecture.adoc[Architecture] @@ -214,6 +215,7 @@ *** xref:oc4:ROOT:how-tos/cloudscale/replace-storage-node.adoc[] *** xref:oc4:ROOT:how-tos/cloudscale/enable-loadbalancer-service.adoc[Enable LoadBalancer Services] +*** xref:oc4:ROOT:how-tos/cloudscale/rotate-api-tokens.adoc[Rotate API Tokens] ** Exoscale // Node management