Clean up clusters on PR completion

This change moves the cleanup of clusters into a dedicated Github
Actions workflow executed when a PR is completed (either through a merge
or by being closed), allowing failed end-to-end test runs to be
inspected.

The actual cleanup is carried out by a small script that we bundle with
the runner image (as that one already comes with doctl). It depends on
clusters having been tagged with a branch identifier, for which we
leverage the cluster name suffix already employed to guarantee unique
cluster names.

.github/workflows/cleanup.yaml

@@ -0,0 +1,25 @@
+name: cleanup-clusters
+
+env:
+  DOCKER_ORG: digitalocean
+
+on:
+  pull_request:
+    types: [closed]
+
+jobs:
+  cleanup-clusters:
+    name: Clean up clusters from the PR
+    runs-on: ubuntu-latest
+    steps:
+      - name: cleanup
+        env:
+          DIGITALOCEAN_ACCESS_TOKEN: ${{ secrets.CSIDigitalOceanAccessToken }}
+          BRANCH: ${{ github.head_ref }}
+        run: |
+          set -eu
+          # Variable definitions and runner image must match things in test.yaml.
+
+          BRANCH="$(echo -n ${BRANCH} | tr -c '[:alnum:]._-' '-')"
+          TAG_IDENTIFIER="$(echo -n ${BRANCH} | sha256sum | cut -c1-7)"
+          docker run --rm -e DIGITALOCEAN_ACCESS_TOKEN --entrypoint /cleanup-clusters.sh ${DOCKER_ORG}/k8s-e2e-test-runner:${BRANCH}-latest "branch-identifier:${TAG_IDENTIFIER}"

.github/workflows/test.yaml

@@ -132,7 +132,7 @@ jobs:
             TAG=latest
           fi
 
-          TIMEOUT=60m make test-e2e E2E_ARGS="-ginkgo-nodes ${NUM_GINKGO_NODES} -driver-image ${DOCKER_ORG}/do-csi-plugin-dev:${TAG} -runner-image ${DOCKER_ORG}/k8s-e2e-test-runner:${RUNNER_IMAGE_TAG_PREFIX}latest -name-suffix ${NAME_SUFFIX} ${{ matrix.kube-release }}"
+          TIMEOUT=60m make test-e2e E2E_ARGS="-ginkgo-nodes ${NUM_GINKGO_NODES} -driver-image ${DOCKER_ORG}/do-csi-plugin-dev:${TAG} -runner-image ${DOCKER_ORG}/k8s-e2e-test-runner:${RUNNER_IMAGE_TAG_PREFIX}latest -name-suffix ${NAME_SUFFIX} -retain ${{ matrix.kube-release }}"
 
   tag-new-master-image:
     runs-on: ubuntu-18.04

test/e2e/Dockerfile

@@ -75,6 +75,9 @@ COPY --from=tools /tini /sbin/
 COPY --from=tools /doctl /usr/local/bin/
 COPY --from=tools /ginkgo /usr/local/bin/
 COPY --from=tools /kubectl /usr/local/bin/
+
+COPY cleanup-clusters.sh /
+
 # Docker comes with built-in tini support (--init parameter) but does not allow
 # to enable child process group killing
 # (https://github.com/krallin/tini#process-group-killing) via "-g". We need this

test/e2e/cleanup-clusters.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 DigitalOcean
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### Delete clusters that match a given DO tag along with all managed volumes that
+### may still exist.
+
+set -o errexit
+set -o nounset
+
+if ! type doctl > /dev/null 2>&1; then
+  echo "doctl is missing" >&2
+  exit 1
+fi
+
+if [[ $# -ne 1 ]]; then
+  echo "usage: $(basename "$0") <identifying cluster tag>" >&2
+  exit 1
+fi
+
+readonly CLUSTER_TAG="$1"
+
+echo "Cleaning up clusters based on cluster tag: ${CLUSTER_TAG}"
+
+while read -r cluster_uuid; do
+  if doctl kubernetes cluster get "${cluster_uuid}" --no-header --format Tags | grep -q "${CLUSTER_TAG}"; then
+    echo "Deleting cluster ${cluster_uuid}"
+    doctl kubernetes cluster delete -f "${cluster_uuid}"
+    echo "Deleting volumes for cluster ${cluster_uuid}"
+    doctl compute volume list --no-header --format ID,Tags | grep "k8s:${cluster_uuid}" | awk '{print $1}' | xargs -n 1 doctl compute volume delete -f
+  fi
+done < <(doctl kubernetes cluster list --no-header --format ID)   # --format field "Tags" does not work on the "list" command
+echo "Done."

test/e2e/e2e_test.go

@@ -388,7 +388,7 @@ func createCluster(ctx context.Context, client *godo.Client, nameSuffix, kubeMaj
 		Name:        clusterName,
 		RegionSlug:  "fra1",
 		VersionSlug: versionSlug,
-		Tags:        []string{"csi-e2e-test", versionTag},
+		Tags:        []string{"csi-e2e-test", versionTag, fmt.Sprintf("branch-identifier:%s", nameSuffix)},
 		NodePools: []*godo.KubernetesNodePoolCreateRequest{
 			{
 				Name:  clusterName + "-pool",