From f325a962a8555901f565f1732ae64c9cf2b3d356 Mon Sep 17 00:00:00 2001 From: Brandon Squizzato <35474886+bsquizz@users.noreply.github.com> Date: Fri, 14 Jan 2022 21:44:06 -0500 Subject: [PATCH] Tear down both unit test ns and smoke test ns (#170) * Tear down both unit test ns and smoke test ns * Simplify oc get logic since namespaces are always fresh now --- cicd/_common_deploy_logic.sh | 61 ++++++++++++++++-------------------- cicd/bootstrap.sh | 2 +- cicd/deploy_ephemeral_db.sh | 1 + cicd/deploy_ephemeral_env.sh | 2 ++ 4 files changed, 31 insertions(+), 35 deletions(-) diff --git a/cicd/_common_deploy_logic.sh b/cicd/_common_deploy_logic.sh index bd5abb6b..6816e07c 100644 --- a/cicd/_common_deploy_logic.sh +++ b/cicd/_common_deploy_logic.sh @@ -21,48 +21,37 @@ set -e : ${COMPONENTS_W_RESOURCES:=""} : ${DEPLOY_TIMEOUT:="600"} K8S_ARTIFACTS_DIR="$ARTIFACTS_DIR/k8s_artifacts/" -START_TIME=$(date +%s) TEARDOWN_RAN=0 -# adapted from https://stackoverflow.com/a/62475429 -# get all events that were emitted at a time greater than $START_TIME, sort by time, and tabulate -function get_oc_events { - { - echo $'TIME\tNAMESPACE\tTYPE\tREASON\tOBJECT\tSOURCE\tMESSAGE'; - oc get events -n $NAMESPACE -o json "$@" | jq -r --argjson start_time "$START_TIME" \ - '.items | - map(. + {t: (.eventTime//.lastTimestamp)}) | - [ .[] | select(.t | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601 > $start_time) ] | - sort_by(.t)[] | - [.t, .metadata.namespace, .type, .reason, .involvedObject.kind + "/" + .involvedObject.name, .source.component + "," + (.source.host//"-"), .message] | - @tsv' - } | column -s $'\t' -t > $K8S_ARTIFACTS_DIR/oc_events.txt -} - -function get_pod_logs { - LOGS_DIR="$K8S_ARTIFACTS_DIR/logs" +function get_pod_logs() { + local ns=$1 + LOGS_DIR="$K8S_ARTIFACTS_DIR/$ns/logs" mkdir -p $LOGS_DIR # get array of pod_name:container1,container2,..,containerN for all containers in all pods - PODS_CONTAINERS=($(oc get pods --ignore-not-found=true -n $NAMESPACE -o "jsonpath={range .items[*]}{' '}{.metadata.name}{':'}{range .spec['containers', 'initContainers'][*]}{.name}{','}")) + echo "Collecting container logs..." + PODS_CONTAINERS=($(oc get pods --ignore-not-found=true -n $ns -o "jsonpath={range .items[*]}{' '}{.metadata.name}{':'}{range .spec['containers', 'initContainers'][*]}{.name}{','}")) for pc in ${PODS_CONTAINERS[@]}; do # https://stackoverflow.com/a/4444841 POD=${pc%%:*} CONTAINERS=${pc#*:} for container in ${CONTAINERS//,/ }; do - oc logs $POD -c $container -n $NAMESPACE > $LOGS_DIR/${POD}_${container}.log || continue - oc logs $POD -c $container --previous -n $NAMESPACE > $LOGS_DIR/${POD}_${container}-previous.log || continue + oc logs $POD -c $container -n $ns > $LOGS_DIR/${POD}_${container}.log 2> /dev/null || continue + oc logs $POD -c $container --previous -n $ns > $LOGS_DIR/${POD}_${container}-previous.log 2> /dev/null || continue done done } -function collect_k8s_artifacts { - mkdir -p $K8S_ARTIFACTS_DIR - get_pod_logs - get_oc_events - oc get all -n $NAMESPACE -o yaml > $K8S_ARTIFACTS_DIR/oc_get_all.yaml - oc get clowdapp -n $NAMESPACE -o yaml > $K8S_ARTIFACTS_DIR/oc_get_clowdapp.yaml - oc get clowdenvironment env-$NAMESPACE -o yaml > $K8S_ARTIFACTS_DIR/oc_get_clowdenvironment.yaml - oc get clowdjobinvocation -n $NAMESPACE -o yaml > $K8S_ARTIFACTS_DIR/oc_get_clowdjobinvocation.yaml +function collect_k8s_artifacts() { + local ns=$1 + DIR="$K8S_ARTIFACTS_DIR/$ns" + mkdir -p $DIR + get_pod_logs $ns + echo "Collecting events and k8s configs..." + oc get events -n $ns --sort-by='.lastTimestamp' > $DIR/oc_get_events.txt + oc get all -n $ns -o yaml > $DIR/oc_get_all.yaml + oc get clowdapp -n $ns -o yaml > $DIR/oc_get_clowdapp.yaml + oc get clowdenvironment env-$ns -o yaml > $DIR/oc_get_clowdenvironment.yaml + oc get clowdjobinvocation -n $ns -o yaml > $DIR/oc_get_clowdjobinvocation.yaml } function teardown { @@ -70,14 +59,18 @@ function teardown { echo "------------------------" echo "----- TEARING DOWN -----" echo "------------------------" - if [ ! -z "$NAMESPACE" ]; then + local ns + RESERVED_NAMESPACES="$DB_NAMESPACE $SMOKE_NAMESPACE" + for ns in $RESERVED_NAMESPACES; do + echo "Running teardown for ns: $ns" set +e - collect_k8s_artifacts + collect_k8s_artifacts $ns if [ "${RELEASE_NAMESPACE:-true}" != "false" ]; then - bonfire namespace release $NAMESPACE -f + echo "Releasing namespace reservation" + bonfire namespace release $ns -f fi - fi - set -e + set -e + done TEARDOWN_RAN=1 } diff --git a/cicd/bootstrap.sh b/cicd/bootstrap.sh index ec121767..e64aab73 100644 --- a/cicd/bootstrap.sh +++ b/cicd/bootstrap.sh @@ -61,5 +61,5 @@ pip install --upgrade pip 'setuptools<58' wheel pip install --upgrade crc-bonfire # clone repo to download cicd scripts -git clone --branch master https://github.com/RedHatInsights/bonfire.git $BONFIRE_ROOT +git clone --branch tear_down_both_namespaces https://github.com/RedHatInsights/bonfire.git $BONFIRE_ROOT diff --git a/cicd/deploy_ephemeral_db.sh b/cicd/deploy_ephemeral_db.sh index cb619b94..1acd3762 100644 --- a/cicd/deploy_ephemeral_db.sh +++ b/cicd/deploy_ephemeral_db.sh @@ -10,6 +10,7 @@ DB_DEPLOYMENT_NAME="${DB_DEPLOYMENT_NAME:-$COMPONENT_NAME-db}" # Deploy k8s resources for app without its dependencies export BONFIRE_NS_REQUESTER="${JOB_NAME}-${BUILD_NUMBER}-db" NAMESPACE=$(bonfire namespace reserve) +DB_NAMESPACE=$NAMESPACE # track which namespace was used here for 'teardown' in common_deploy_logic # TODO: add code to bonfire to deploy an app if it is defined in 'sharedAppDbName' on the ClowdApp # TODO: add a bonfire command to deploy just an app's DB set -x diff --git a/cicd/deploy_ephemeral_env.sh b/cicd/deploy_ephemeral_env.sh index 7ce33913..1739c370 100644 --- a/cicd/deploy_ephemeral_env.sh +++ b/cicd/deploy_ephemeral_env.sh @@ -9,6 +9,8 @@ source ${CICD_ROOT}/_common_deploy_logic.sh set -x export BONFIRE_NS_REQUESTER="${JOB_NAME}-${BUILD_NUMBER}" export NAMESPACE=$(bonfire namespace reserve) +SMOKE_NAMESPACE=$NAMESPACE # track which namespace was used here for 'teardown' in common_deploy_logic + bonfire deploy \ ${APP_NAME} \ --source=appsre \