Tear down both unit test ns and smoke test ns (#170)

* Tear down both unit test ns and smoke test ns

* Simplify oc get logic since namespaces are always fresh now

cicd/_common_deploy_logic.sh

@@ -21,63 +21,56 @@ set -e
 : ${COMPONENTS_W_RESOURCES:=""}
 : ${DEPLOY_TIMEOUT:="600"}
 K8S_ARTIFACTS_DIR="$ARTIFACTS_DIR/k8s_artifacts/"
-START_TIME=$(date +%s)
 TEARDOWN_RAN=0
 
-# adapted from https://stackoverflow.com/a/62475429
-# get all events that were emitted at a time greater than $START_TIME, sort by time, and tabulate
-function get_oc_events {
-    {
-        echo $'TIME\tNAMESPACE\tTYPE\tREASON\tOBJECT\tSOURCE\tMESSAGE';
-        oc get events -n $NAMESPACE -o json "$@" | jq -r --argjson start_time "$START_TIME" \
-            '.items |
-            map(. + {t: (.eventTime//.lastTimestamp)}) |
-            [ .[] | select(.t | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601 > $start_time) ] |
-            sort_by(.t)[] |
-            [.t, .metadata.namespace, .type, .reason, .involvedObject.kind + "/" + .involvedObject.name, .source.component + "," + (.source.host//"-"), .message] |
-            @tsv'
-    } | column -s $'\t' -t > $K8S_ARTIFACTS_DIR/oc_events.txt
-}
-
-function get_pod_logs {
-    LOGS_DIR="$K8S_ARTIFACTS_DIR/logs"
+function get_pod_logs() {
+    local ns=$1
+    LOGS_DIR="$K8S_ARTIFACTS_DIR/$ns/logs"
     mkdir -p $LOGS_DIR
     # get array of pod_name:container1,container2,..,containerN for all containers in all pods
-    PODS_CONTAINERS=($(oc get pods --ignore-not-found=true -n $NAMESPACE -o "jsonpath={range .items[*]}{' '}{.metadata.name}{':'}{range .spec['containers', 'initContainers'][*]}{.name}{','}"))
+    echo "Collecting container logs..."
+    PODS_CONTAINERS=($(oc get pods --ignore-not-found=true -n $ns -o "jsonpath={range .items[*]}{' '}{.metadata.name}{':'}{range .spec['containers', 'initContainers'][*]}{.name}{','}"))
     for pc in ${PODS_CONTAINERS[@]}; do
         # https://stackoverflow.com/a/4444841
         POD=${pc%%:*}
         CONTAINERS=${pc#*:}
         for container in ${CONTAINERS//,/ }; do
-            oc logs $POD -c $container -n $NAMESPACE > $LOGS_DIR/${POD}_${container}.log || continue
-            oc logs $POD -c $container --previous -n $NAMESPACE > $LOGS_DIR/${POD}_${container}-previous.log || continue
+            oc logs $POD -c $container -n $ns > $LOGS_DIR/${POD}_${container}.log 2> /dev/null || continue
+            oc logs $POD -c $container --previous -n $ns > $LOGS_DIR/${POD}_${container}-previous.log 2> /dev/null || continue
         done
     done
 }
 
-function collect_k8s_artifacts {
-    mkdir -p $K8S_ARTIFACTS_DIR
-    get_pod_logs
-    get_oc_events
-    oc get all -n $NAMESPACE -o yaml > $K8S_ARTIFACTS_DIR/oc_get_all.yaml
-    oc get clowdapp -n $NAMESPACE -o yaml > $K8S_ARTIFACTS_DIR/oc_get_clowdapp.yaml
-    oc get clowdenvironment env-$NAMESPACE -o yaml > $K8S_ARTIFACTS_DIR/oc_get_clowdenvironment.yaml
-    oc get clowdjobinvocation -n $NAMESPACE -o yaml > $K8S_ARTIFACTS_DIR/oc_get_clowdjobinvocation.yaml
+function collect_k8s_artifacts() {
+    local ns=$1
+    DIR="$K8S_ARTIFACTS_DIR/$ns"
+    mkdir -p $DIR
+    get_pod_logs $ns
+    echo "Collecting events and k8s configs..."
+    oc get events -n $ns --sort-by='.lastTimestamp' > $DIR/oc_get_events.txt
+    oc get all -n $ns -o yaml > $DIR/oc_get_all.yaml
+    oc get clowdapp -n $ns -o yaml > $DIR/oc_get_clowdapp.yaml
+    oc get clowdenvironment env-$ns -o yaml > $DIR/oc_get_clowdenvironment.yaml
+    oc get clowdjobinvocation -n $ns -o yaml > $DIR/oc_get_clowdjobinvocation.yaml
 }
 
 function teardown {
     [ "$TEARDOWN_RAN" -ne "0" ] && return
     echo "------------------------"
     echo "----- TEARING DOWN -----"
     echo "------------------------"
-    if [ ! -z "$NAMESPACE" ]; then
+    local ns
+    RESERVED_NAMESPACES="$DB_NAMESPACE $SMOKE_NAMESPACE"
+    for ns in $RESERVED_NAMESPACES; do
+        echo "Running teardown for ns: $ns"
         set +e
-        collect_k8s_artifacts
+        collect_k8s_artifacts $ns
         if [ "${RELEASE_NAMESPACE:-true}" != "false" ]; then
-            bonfire namespace release $NAMESPACE -f
+            echo "Releasing namespace reservation"
+            bonfire namespace release $ns -f
         fi
-    fi
-    set -e
+        set -e
+    done
     TEARDOWN_RAN=1
 }
 

cicd/bootstrap.sh

@@ -61,5 +61,5 @@ pip install --upgrade pip 'setuptools<58' wheel
 pip install --upgrade crc-bonfire
 
 # clone repo to download cicd scripts
-git clone --branch master https://github.com/RedHatInsights/bonfire.git $BONFIRE_ROOT
+git clone --branch tear_down_both_namespaces https://github.com/RedHatInsights/bonfire.git $BONFIRE_ROOT
 

cicd/deploy_ephemeral_db.sh

@@ -10,6 +10,7 @@ DB_DEPLOYMENT_NAME="${DB_DEPLOYMENT_NAME:-$COMPONENT_NAME-db}"
 # Deploy k8s resources for app without its dependencies
 export BONFIRE_NS_REQUESTER="${JOB_NAME}-${BUILD_NUMBER}-db"
 NAMESPACE=$(bonfire namespace reserve)
+DB_NAMESPACE=$NAMESPACE  # track which namespace was used here for 'teardown' in common_deploy_logic
 # TODO: add code to bonfire to deploy an app if it is defined in 'sharedAppDbName' on the ClowdApp
 # TODO: add a bonfire command to deploy just an app's DB
 set -x

cicd/deploy_ephemeral_env.sh

@@ -9,6 +9,8 @@ source ${CICD_ROOT}/_common_deploy_logic.sh
 set -x
 export BONFIRE_NS_REQUESTER="${JOB_NAME}-${BUILD_NUMBER}"
 export NAMESPACE=$(bonfire namespace reserve)
+SMOKE_NAMESPACE=$NAMESPACE  # track which namespace was used here for 'teardown' in common_deploy_logic
+
 bonfire deploy \
     ${APP_NAME} \
     --source=appsre \