diff --git a/bonfire/openshift.py b/bonfire/openshift.py index a49b14d2..1b429fe8 100644 --- a/bonfire/openshift.py +++ b/bonfire/openshift.py @@ -4,6 +4,7 @@ import re import threading import time +import sys import sh from sh import ErrorReturnCode, TimeoutException @@ -112,27 +113,32 @@ def _get_logging_args(args, kwargs): def _exec_oc(*args, **kwargs): + _print = kwargs.pop("_print", False) _silent = kwargs.pop("_silent", False) _ignore_immutable = kwargs.pop("_ignore_immutable", True) _retry_conflicts = kwargs.pop("_retry_conflicts", True) _retry_io_errors = kwargs.pop("_retry_io_errors", True) _stdout_log_prefix = kwargs.pop("_stdout_log_prefix", " |stdout| ") _stderr_log_prefix = kwargs.pop("_stderr_log_prefix", " |stderr| ") - kwargs["_bg"] = True kwargs["_bg_exc"] = False + # define stdout/stderr callback funcs err_lines = [] out_lines = [] def _err_line_handler(line, _, process): threading.current_thread().name = f"pid-{process.pid}" + if _print: + print(line.rstrip(), file=sys.stderr) if not _silent: log.info("%s%s", _stderr_log_prefix, line.rstrip()) err_lines.append(line) def _out_line_handler(line, _, process): threading.current_thread().name = f"pid-{process.pid}" + if _print: + print(line.rstrip()) if not _silent: log.info("%s%s", _stdout_log_prefix, line.rstrip()) out_lines.append(line) @@ -208,7 +214,8 @@ def oc(*args, **kwargs): Optional kwargs: _ignore_errors: if ErrorReturnCode is hit, don't re-raise it (default False) - _silent: don't print command or resulting stdout (default False) + _silent: don't log command or resulting output (default False) + _print: print stdout/stderr output directly to stdout/stderr (default False) _ignore_immutable: ignore errors related to immutable objects (default True) _retry_conflicts: retry commands if a conflict error is hit _retry_io_errors: retry commands if i/o error is hit diff --git a/cicd/_common_deploy_logic.sh b/cicd/_common_deploy_logic.sh index bb64a894..1ffcc6a1 100644 --- a/cicd/_common_deploy_logic.sh +++ b/cicd/_common_deploy_logic.sh @@ -29,14 +29,14 @@ function get_pod_logs() { mkdir -p $LOGS_DIR # get array of pod_name:container1,container2,..,containerN for all containers in all pods echo "Collecting container logs..." - PODS_CONTAINERS=($(oc get pods --ignore-not-found=true -n $ns -o "jsonpath={range .items[*]}{' '}{.metadata.name}{':'}{range .spec['containers', 'initContainers'][*]}{.name}{','}")) + PODS_CONTAINERS=($(oc_wrapper get pods --ignore-not-found=true -n $ns -o "jsonpath={range .items[*]}{' '}{.metadata.name}{':'}{range .spec['containers', 'initContainers'][*]}{.name}{','}")) for pc in ${PODS_CONTAINERS[@]}; do # https://stackoverflow.com/a/4444841 POD=${pc%%:*} CONTAINERS=${pc#*:} for container in ${CONTAINERS//,/ }; do - oc logs $POD -c $container -n $ns > $LOGS_DIR/${POD}_${container}.log 2> /dev/null || continue - oc logs $POD -c $container --previous -n $ns > $LOGS_DIR/${POD}_${container}-previous.log 2> /dev/null || continue + oc_wrapper logs $POD -c $container -n $ns > $LOGS_DIR/${POD}_${container}.log 2> /dev/null || continue + oc_wrapper logs $POD -c $container --previous -n $ns > $LOGS_DIR/${POD}_${container}-previous.log 2> /dev/null || continue done done } @@ -47,11 +47,11 @@ function collect_k8s_artifacts() { mkdir -p $DIR get_pod_logs $ns echo "Collecting events and k8s configs..." - oc get events -n $ns --sort-by='.lastTimestamp' > $DIR/oc_get_events.txt - oc get all -n $ns -o yaml > $DIR/oc_get_all.yaml - oc get clowdapp -n $ns -o yaml > $DIR/oc_get_clowdapp.yaml - oc get clowdenvironment env-$ns -o yaml > $DIR/oc_get_clowdenvironment.yaml - oc get clowdjobinvocation -n $ns -o yaml > $DIR/oc_get_clowdjobinvocation.yaml + oc_wrapper get events -n $ns --sort-by='.lastTimestamp' > $DIR/oc_get_events.txt + oc_wrapper get all -n $ns -o yaml > $DIR/oc_get_all.yaml + oc_wrapper get clowdapp -n $ns -o yaml > $DIR/oc_get_clowdapp.yaml + oc_wrapper get clowdenvironment env-$ns -o yaml > $DIR/oc_get_clowdenvironment.yaml + oc_wrapper get clowdjobinvocation -n $ns -o yaml > $DIR/oc_get_clowdjobinvocation.yaml } function teardown { diff --git a/cicd/bin/oc_wrapper b/cicd/bin/oc_wrapper new file mode 100755 index 00000000..77b69e97 --- /dev/null +++ b/cicd/bin/oc_wrapper @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +import sys +import logging + +from sh import ErrorReturnCode +from bonfire.openshift import oc + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + args = sys.argv[1:] + try: + oc(*args, _silent=True, _print=True) + except ErrorReturnCode as err: + sys.exit(err.exit_code) diff --git a/cicd/bootstrap.sh b/cicd/bootstrap.sh index 0fd5c37a..821b6ac1 100644 --- a/cicd/bootstrap.sh +++ b/cicd/bootstrap.sh @@ -63,48 +63,14 @@ python3 -m venv .bonfire_venv source .bonfire_venv/bin/activate pip install --upgrade pip 'setuptools<58' wheel -pip install --upgrade crc-bonfire +pip install --upgrade 'crc-bonfire>=3.6.0' # clone repo to download cicd scripts rm -fr $BONFIRE_ROOT git clone --branch master https://github.com/RedHatInsights/bonfire.git $BONFIRE_ROOT -# Add a retry mechanism to 'oc' command calls -oc_wrapper() { - # hide all the extra stuff we're doing in here if user called 'set -x' before 'oc' - # to make debugging log output easier - # https://stackoverflow.com/a/50668339 - if [ -o xtrace ]; then - set +x - trap 'set -x' RETURN - fi - - real_oc=$(which oc) - retries=3 - backoff=3 - attempt=0 - - if [ -z "$real_oc" ]; then - echo "ERROR: unable to locate 'oc' command on PATH" >&2 - return 1 - fi - - while true; do - attempt=$((attempt+1)) - $real_oc $@ && return 0 # exit here if 'oc' completes successfully - - if [ "$attempt" -lt $retries ]; then - sleep_time=$(($attempt*$backoff)) - echo "oc command hit error (attempt $attempt/$retries), retrying in $sleep_time sec" >&2 - sleep $sleep_time - else - break - fi - done - - echo "oc command failed, gave up after $retries tries" >&2 - return 1 -} +# Gives access to helper commands such as "oc_wrapper" +export PATH=$PATH:${CICD_ROOT}/bin # log in to ephemeral cluster oc_wrapper login --token=$OC_LOGIN_TOKEN --server=$OC_LOGIN_SERVER diff --git a/cicd/cji_smoke_test.sh b/cicd/cji_smoke_test.sh index 62637e9c..394d609d 100644 --- a/cicd/cji_smoke_test.sh +++ b/cicd/cji_smoke_test.sh @@ -57,7 +57,7 @@ oc_wrapper logs -n $NAMESPACE $POD -f & # Wait for the job to Complete or Fail before we try to grab artifacts # condition=complete does trigger when the job fails set -x -oc wait --timeout=$IQE_CJI_TIMEOUT --for=condition=JobInvocationComplete -n $NAMESPACE cji/$CJI_NAME +oc_wrapper wait --timeout=$IQE_CJI_TIMEOUT --for=condition=JobInvocationComplete -n $NAMESPACE cji/$CJI_NAME set +x # Log in again -- perhaps this will fix the intermittent 'service account lost permissions' error?