Debug log membership verification

asmcli/lib/checks.sh

@@ -88,23 +88,28 @@ is_managed_cas_installed() {
 }
 
 is_cluster_registered() {
+  debug "is_cluster_registered()"
   local VERIFIED_REGISTRATION; VERIFIED_REGISTRATION="$(context_get-option "VERIFIED_REGISTRATION")"
   if [[ "${VERIFIED_REGISTRATION}" -eq 1 ]]; then return; fi
 
   info "Verifying cluster registration."
 
   if ! is_membership_crd_installed; then
+    debug "Couldn't find membership CRD."
     false
     return
   fi
 
   local MEMBERSHIP_DATA IDENTITY_PROVIDER
   MEMBERSHIP_DATA="$(retry 2 kubectl get memberships.hub.gke.io membership -ojson 2>/dev/null)"
+  debug "${MEMBERSHIP_DATA}"
 
   # expected value is the project id to which the cluster is registered
   IDENTITY_PROVIDER="$(echo "${MEMBERSHIP_DATA}" \
     | jq .spec.identity_provider \
     | sed -E 's/.*projects\/|\/locations.*//g')"
+  debug "${IDENTITY_PROVIDER}"
+
   if [[ -z "${IDENTITY_PROVIDER}" || "${IDENTITY_PROVIDER}" == 'null' ]]; then
     { read -r -d '' MSG; fatal "${MSG}"; } <<EOF || true
 Cluster has memberships.hub.gke.io CRD but no identity provider specified.
@@ -123,12 +128,16 @@ EOF
   LOCATION="$(echo "${MEMBERSHIP_DATA}" \
     | jq -r .spec.owner.id \
     | sed -E 's/.*locations\/|\/memberships.*//g')"
+  debug "${LOCATION}"
   MEMBERSHIP="$(echo "${MEMBERSHIP_DATA}" \
     | jq -r .spec.owner.id \
     | sed -E 's/.*memberships\///g')"
+  debug "${MEMBERSHIP}"
   WANT="name.*projects/${FLEET_ID}/locations/${LOCATION}/memberships/${MEMBERSHIP}"
   G_DATA="$(gcloud container hub memberships list --project "${FLEET_ID}" --format=json)"
+  debug "${G_DATA}"
   LIST="$(echo "${G_DATA}" | grep "${WANT}")"
+  debug "${LIST}"
 
   if [[ "${IDENTITY_PROVIDER}" != "${FLEET_ID}" ]] && \
      [[ "${IDENTITY_PROVIDER}" != "${FLEET_HOST_PROJECT_NUMBER}" ]] || \
@@ -183,20 +192,22 @@ is_workload_identity_enabled() {
 
 is_membership_crd_installed() {
   local OUTPUT
-  if ! OUTPUT="$(retry 2 kubectl get crd memberships.hub.gke.io -ojsonpath="{..metadata.name}" 2>/dev/null)"; then
+  if ! OUTPUT="$(kubectl get crd memberships.hub.gke.io -ojsonpath="{..metadata.name}" 2>/dev/null)"; then
     false
     return
   fi
+  debug "${OUTPUT}"
 
   if [[ "$(echo "${OUTPUT}" | grep -w -c memberships || true)" -eq 0 ]]; then
     false
     return
   fi
 
-  if ! OUTPUT="$(retry 2 kubectl get memberships.hub.gke.io -ojsonpath="{..metadata.name}" 2>/dev/null)"; then
+  if ! OUTPUT="$(kubectl get memberships.hub.gke.io -ojsonpath="{..metadata.name}" 2>/dev/null)"; then
     false
     return
   fi
+  debug "${OUTPUT}"
 
   if [[ "$(echo "${OUTPUT}" | grep -w -c membership || true)" -eq 0 ]]; then
     false

asmcli/lib/util.sh

@@ -49,6 +49,8 @@ run_command() {
 # re-get credentials in case something caused the k8s IP to change.
 #######
 retry() {
+  local CMD="'$*'"
+  debug "Attempting to run \`${CMD}\`."
   local MAX_TRIES; MAX_TRIES="${1}";
   shift 1
   for i in $(seq 0 "${MAX_TRIES}"); do
@@ -59,7 +61,6 @@ retry() {
     warn "Failed, retrying...($((i+1)) of ${MAX_TRIES})"
     sleep 2
   done
-  local CMD="'$*'"
   warn "Command $CMD failed."
   false
 }

asmcli/tests/common.sh

@@ -292,11 +292,11 @@ create_working_cluster() {
 
   KUBECONFIG="$(mktemp)"
   export KUBECONFIG
-  configure_kubectl  "${CLUSTER_NAME}" "${PROJECT_ID}" "${CLUSTER_LOCATION}"
+  configure_test_kubectl  "${CLUSTER_NAME}" "${PROJECT_ID}" "${CLUSTER_LOCATION}"
 
 }
 
-configure_kubectl() {
+configure_test_kubectl() {
   local CLUSTER_NAME; CLUSTER_NAME="${1}";
   local PROJECT_ID; PROJECT_ID="${2}";
   local CLUSTER_LOCATION; CLUSTER_LOCATION="${3}";
@@ -631,7 +631,7 @@ run_required_role() {
 
   OUTPUT_DIR="$(mktemp -d)"
 
-  configure_kubectl "${LT_CLUSTER_NAME}" "${PROJECT_ID}" "${LT_CLUSTER_LOCATION}"
+  configure_test_kubectl "${LT_CLUSTER_NAME}" "${PROJECT_ID}" "${LT_CLUSTER_LOCATION}"
 
   if [[ -n "${KEY_FILE}" && -n "${SERVICE_ACCOUNT}" ]]; then
     KEY_FILE="-k ${KEY_FILE}"
@@ -700,7 +700,7 @@ run_basic_test() {
   LT_NAMESPACE="$(uniq_name "${SCRIPT_NAME}" "${BUILD_ID}")"
   OUTPUT_DIR="${OUTPUT_DIR:=$(mktemp -d)}"
 
-  configure_kubectl "${LT_CLUSTER_NAME}" "${PROJECT_ID}" "${LT_CLUSTER_LOCATION}"
+  configure_test_kubectl "${LT_CLUSTER_NAME}" "${PROJECT_ID}" "${LT_CLUSTER_LOCATION}"
 
   trap 'remove_ns "${LT_NAMESPACE}"; rm "${LT_NAMESPACE}"; exit 1' ERR
 

asmcli/tests/run_create_mesh

@@ -43,7 +43,7 @@ EOF
   # Test starts here
   echo "Validating the clusters are unregistered..."
   while read -r CLUSTER_NAME; do
-    configure_kubectl "${CLUSTER_NAME}" "${PROJECT_ID}" "${CLUSTER_LOCATION}"
+    configure_test_kubectl "${CLUSTER_NAME}" "${PROJECT_ID}" "${CLUSTER_LOCATION}"
     if is_cluster_registered "${CLUSTER_NAME}"; then
       echo "Cluster ${CLUSTER_NAME} is already registered before the test to ${PROJECT_ID}".
       exit 2
@@ -82,7 +82,7 @@ set_up_clusters() {
 
   local CLUSTER_NAME
   while read -r CLUSTER_NAME; do
-    configure_kubectl "${CLUSTER_NAME}" "${PROJECT_ID}" "${CLUSTER_LOCATION}"
+    configure_test_kubectl "${CLUSTER_NAME}" "${PROJECT_ID}" "${CLUSTER_LOCATION}"
 
     create_ns "${ISTIO_NAMESPACE}"
 
@@ -110,7 +110,7 @@ EOF
 clean_up_clusters() {
   local CLUSTER_NAME
   while read -r CLUSTER_NAME; do
-    configure_kubectl "${CLUSTER_NAME}" "${PROJECT_ID}" "${CLUSTER_LOCATION}"
+    configure_test_kubectl "${CLUSTER_NAME}" "${PROJECT_ID}" "${CLUSTER_LOCATION}"
     unregister_cluster_if_possible "${CLUSTER_NAME}" "${PROJECT_ID}" "${CLUSTER_LOCATION}"
     cleanup_lt_cluster "${LT_NAMESPACE}" "${OUTPUT_DIR}"
   done <<EOF

asmcli/tests/setup_longterm_cluster

@@ -28,7 +28,7 @@ setup_cluster() {
       -p "${LT_PROJECT_ID}" \
       -D "${OUTPUT_DIR}" > /dev/null
 
-    configure_kubectl "${cluster}" "${LT_PROJECT_ID}" "${LT_CLUSTER_LOCATION}"
+    configure_test_kubectl "${cluster}" "${LT_PROJECT_ID}" "${LT_CLUSTER_LOCATION}"
     kubectl delete validatingwebhookconfiguration istiod-istio-system || true
     cleanup_old_test_namespaces "${OUTPUT_DIR}"
   else
@@ -66,6 +66,7 @@ setup_cluster() {
 
   echo "Performing necessary cluster setup on ${cluster}"
 
+  _DEBUG=1 \
   _CI_I_AM_A_TEST_ROBOT=1 \
   ../asmcli \
     install \