How-To: Adopt worker nodes with the cloudscale Machine API Provider

docs/modules/ROOT/pages/how-tos/cloudscale/decommission.adoc

@@ -22,7 +22,7 @@ include::partial$cloudscale/prerequisites.adoc[]
 [source,bash]
 ----
 export CLUSTER_ID=<lieutenant-cluster-id>
-export GITLAB_TOKEN=<gitlab-api-token> # From https://git.vshn.net/-/profile/personal_access_tokens
+export GITLAB_TOKEN=<gitlab-api-token> # From https://git.vshn.net/-/user_settings/personal_access_tokens
 export GITLAB_USER=<gitlab-user-name>
 ----
 

docs/modules/ROOT/pages/how-tos/cloudscale/provider-adopt-worker-nodes.adoc

@@ -0,0 +1,230 @@
+= Adopt worker nodes with the cloudscale Machine API Provider
+
+[abstract]
+--
+Steps to adopt worker nodes on https://cloudscale.ch[cloudscale] with the https://github.com/appuio/machine-api-provider-cloudscale[cloudscale Machine API Provider].
+--
+
+== Starting situation
+
+* You already have an OpenShift 4 cluster on cloudscale
+* You have admin-level access to the cluster
+* You want the nodes adopted by the https://github.com/appuio/machine-api-provider-cloudscale[cloudscale Machine API Provider]
+
+== Prerequisites
+
+The following CLI utilities need to be available locally:
+
+* `commodore`, see https://syn.tools/commodore/running-commodore.html[Running Commodore]
+* `docker`
+* `kubectl`
+* `vault`
+* `yq`
+
+== Prepare local environment
+
+include::partial$cloudscale/setup-local-env.adoc[]
+
+== Update Cluster Config
+
+. Update cluster config
++
+[source,bash]
+----
+pushd inventory/classes/"${TENANT_ID}"
+
+yq -i '.applications += "machine-api-provider-cloudscale"' \
+  ${CLUSTER_ID}.yml
+
+yq eval -i ".parameters.openshift4_terraform.terraform_variables.make_worker_adoptable_by_provider = true" \
+  ${CLUSTER_ID}.yml
+yq eval -i '.parameters.machine_api_provider_cloudscale.secrets["cloudscale-user-data"].stringData.ignitionCA = "${openshift4_terraform:terraform_variables:ignition_ca}"' \
+  ${CLUSTER_ID}.yml
+
+git commit -m "Allow adoption of worker nodes" "${CLUSTER_ID}.yml"
+git push
+popd
+----
+
+. Compile and push the cluster catalog.
++
+[source,bash]
+----
+commodore catalog compile "${CLUSTER_ID}" --push
+----
+
+== Prepare Terraform environment
+
+include::partial$cloudscale/configure-terraform-secrets.adoc[]
+
+include::partial$setup_terraform.adoc[]
+
+== Run terraform
+
+. Verify terraform output and apply the changes if everything looks good.
++
+Terraform will tag the nodes as preparation for the adoption by the cloudscale Machine API Provider.
++
+[source,bash]
+----
+terraform apply
+----
+
+== Apply Machine and MachineSet manifests
+
+[IMPORTANT]
+====
+Please ensure the terraform apply has completed successfully before proceeding with this step.
+Without the tags applied by Terraform, nodes will be duplicated with the same name and weird stuff might happen.
+
+Please be careful to not apply the `MachineSet` before the `Machine` manifests.
+====
+
+. Copy `worker-machines_yml` from the Terraform output and apply it to the cluster.
++
+[source,bash]
+----
+terraform output -raw worker-machines_yml | yq -P > worker-machines.yml
+head worker-machines.yml
+kubectl apply -f worker-machines.yml
+----
+
+. Check that all machines are in the `Running` state.
++
+[source,bash]
+----
+kubectl get -f worker-machines.yml
+----
+
+. Copy `worker-machineset_yml` from the Terraform output and apply it to the cluster.
++
+[source,bash]
+----
+terraform output -raw worker-machineset_yml | yq -P > worker-machineset.yml
+head worker-machineset.yml
+kubectl apply -f worker-machineset.yml
+----
+
+. Copy `infra-machines_yml` from the Terraform output and apply it to the cluster.
++
+[source,bash]
+----
+terraform output -raw infra-machines_yml | yq -P > infra-machines.yml
+head infra-machines.yml
+kubectl apply -f infra-machines.yml
+----
+
+. Check that all machines are in the `Running` state.
++
+[source,bash]
+----
+kubectl get -f infra-machines.yml
+----
+
+. Copy `infra-machineset_yml` from the Terraform output and apply it to the cluster.
++
+[source,bash]
+----
+terraform output -raw infra-machineset_yml | yq -P > infra-machineset.yml
+head infra-machineset.yml
+kubectl apply -f infra-machineset.yml
+----
+
+. Check for additional worker groups and apply them if necessary.
++
+[source,bash]
+----
+terraform output -raw additional-worker-machines_yml > /dev/null 2>&1 || echo "No additional worker groups"
+----
+
+. If the output shows "No additional worker groups," jump to <<remove_nodes_tf_state>>.
+
+. Copy `additional-worker-machines_yml` from the Terraform output and apply it to the cluster.
++
+[source,bash]
+----
+terraform output -raw additional-worker-machines_yml | yq -P > additional-worker-machines.yml
+head additional-worker-machines.yml
+kubectl apply -f additional-worker-machines.yml
+----
+
+. Check that all machines are in the `Running` state.
++
+[source,bash]
+----
+kubectl get -f additional-worker-machines.yml
+----
+
+. Copy `additional-worker-machinesets_yml` from the Terraform output and apply it to the cluster.
++
+[source,bash]
+----
+terraform output -raw additional-worker-machinesets_yml | yq -P > additional-worker-machinesets.yml
+head additional-worker-machinesets.yml
+kubectl apply -f additional-worker-machinesets.yml
+----
+
+== Remove nodes from the Terraform state [[remove_nodes_tf_state]]
+
+. Remove the nodes from the Terraform state.
++
+[source,bash]
+----
+terraform state rm module.cluster.module.worker
+terraform state rm module.cluster.module.infra
+terraform state rm module.cluster.module.additional_worker
+cat > override.tf <<EOF
+module "cluster" {
+  infra_count              = 0
+  worker_count             = 0
+  additional_worker_groups = {}
+}
+EOF
+----
+
+. Check the terraform plan output and apply the changes.
+There should be no server recreation.
+Hieradata changes must be ignored, otherwise the cluster ingress controller will become unavailable.
++
+[source,bash]
+----
+terraform plan
+terraform apply
+----
+
+== Cleanup
+
+. Persist the Terraform changes and start managing the machine sets.
++
+[source,bash]
+----
+popd
+pushd "inventory/classes/${TENANT_ID}"
+
+yq -i e '.parameters.openshift4_terraform.terraform_variables.additional_worker_groups= {}' \
+  "${CLUSTER_ID}.yml"
+yq -i e '.parameters.openshift4_terraform.terraform_variables.infra_count = 0' \
+  "${CLUSTER_ID}.yml"
+yq -i e '.parameters.openshift4_terraform.terraform_variables.worker_count = 0' \
+  "${CLUSTER_ID}.yml"
+
+yq -i ea 'select(fileIndex == 0) as $cluster |
+          $cluster.parameters.openshift4_nodes.machineSets =
+            ([select(fileIndex > 0)][] as $ms ireduce ({};
+              $ms.metadata.name as $msn |
+              del($ms.apiVersion) |
+              del($ms.kind) |
+              del($ms.metadata.name) |
+              del($ms.metadata.labels.name) |
+              del($ms.metadata.namespace) |
+              . * {$msn: $ms}
+            )) |
+          $cluster' \
+  "${CLUSTER_ID}.yml" ../../../catalog/manifests/openshift4-terraform/*machineset*.yml
+
+git commit -am "Persist provider adopted machine and terraform state for ${CLUSTER_ID}"
+git push origin master
+popd
+
+commodore catalog compile "${CLUSTER_ID}" --push
+----

docs/modules/ROOT/pages/how-tos/exoscale/decommission.adoc

@@ -39,7 +39,7 @@ export EXOSCALE_ZONE=<cluster-region>  # e.g. ch-gva-2
 
 export CLUSTER_ID=<cluster-name>
 
-# From https://git.vshn.net/-/profile/personal_access_tokens
+# From https://git.vshn.net/-/user_settings/personal_access_tokens
 export GITLAB_TOKEN=<gitlab-api-token>
 export GITLAB_USER=<gitlab-user-name>
 

docs/modules/ROOT/partials/nav.adoc

@@ -162,6 +162,7 @@
 *** xref:oc4:ROOT:how-tos/cloudscale/update_compute_flavors.adoc[Update compute flavors]
 *** xref:oc4:ROOT:how-tos/cloudscale/remove_node.adoc[]
 *** xref:oc4:ROOT:how-tos/cloudscale/increase-worker-node-disk.adoc[]
+*** xref:oc4:ROOT:how-tos/cloudscale/provider-adopt-worker-nodes.adoc[]
 
 ** Exoscale
 *** xref:oc4:ROOT:how-tos/exoscale/remove_node.adoc[]

docs/modules/ROOT/partials/setup_terraform.adoc

@@ -13,8 +13,9 @@ tf_tag=$(\
 
 # Generate the terraform alias
 base_dir=$(pwd)
-alias terraform='docker run -it --rm \
+alias terraform='touch .terraformrc; docker run -it --rm \
   -e REAL_UID=$(id -u) \
+  -e TF_CLI_CONFIG_FILE=/tf/.terraformrc \
   --env-file ${base_dir}/terraform.env \
   -w /tf \
   -v $(pwd):/tf \

docs/modules/ROOT/partials/vshn-input.adoc

@@ -2,7 +2,7 @@ ifeval::["{needs_gitlab}" != "no"]
 .Access to VSHN GitLab
 [source,bash]
 ----
-# From https://git.vshn.net/-/profile/personal_access_tokens, "api" scope is sufficient
+# From https://git.vshn.net/-/user_settings/personal_access_tokens, "api" scope is sufficient
 export GITLAB_TOKEN=<gitlab-api-token>
 export GITLAB_USER=<gitlab-user-name>
 ----