Rewrite oc_wrapper in python (#193)

Add _print kwarg to oc() and make 'oc_wrapper' a python executable

bonfire/openshift.py

@@ -4,6 +4,7 @@
 import re
 import threading
 import time
+import sys
 
 import sh
 from sh import ErrorReturnCode, TimeoutException
@@ -112,27 +113,32 @@ def _get_logging_args(args, kwargs):
 
 
 def _exec_oc(*args, **kwargs):
+    _print = kwargs.pop("_print", False)
     _silent = kwargs.pop("_silent", False)
     _ignore_immutable = kwargs.pop("_ignore_immutable", True)
     _retry_conflicts = kwargs.pop("_retry_conflicts", True)
     _retry_io_errors = kwargs.pop("_retry_io_errors", True)
     _stdout_log_prefix = kwargs.pop("_stdout_log_prefix", " |stdout| ")
     _stderr_log_prefix = kwargs.pop("_stderr_log_prefix", " |stderr| ")
-
     kwargs["_bg"] = True
     kwargs["_bg_exc"] = False
 
+    # define stdout/stderr callback funcs
     err_lines = []
     out_lines = []
 
     def _err_line_handler(line, _, process):
         threading.current_thread().name = f"pid-{process.pid}"
+        if _print:
+            print(line.rstrip(), file=sys.stderr)
         if not _silent:
             log.info("%s%s", _stderr_log_prefix, line.rstrip())
         err_lines.append(line)
 
     def _out_line_handler(line, _, process):
         threading.current_thread().name = f"pid-{process.pid}"
+        if _print:
+            print(line.rstrip())
         if not _silent:
             log.info("%s%s", _stdout_log_prefix, line.rstrip())
         out_lines.append(line)
@@ -208,7 +214,8 @@ def oc(*args, **kwargs):
 
     Optional kwargs:
         _ignore_errors: if ErrorReturnCode is hit, don't re-raise it (default False)
-        _silent: don't print command or resulting stdout (default False)
+        _silent: don't log command or resulting output (default False)
+        _print: print stdout/stderr output directly to stdout/stderr (default False)
         _ignore_immutable: ignore errors related to immutable objects (default True)
         _retry_conflicts: retry commands if a conflict error is hit
         _retry_io_errors: retry commands if i/o error is hit

cicd/_common_deploy_logic.sh

@@ -29,14 +29,14 @@ function get_pod_logs() {
     mkdir -p $LOGS_DIR
     # get array of pod_name:container1,container2,..,containerN for all containers in all pods
     echo "Collecting container logs..."
-    PODS_CONTAINERS=($(oc get pods --ignore-not-found=true -n $ns -o "jsonpath={range .items[*]}{' '}{.metadata.name}{':'}{range .spec['containers', 'initContainers'][*]}{.name}{','}"))
+    PODS_CONTAINERS=($(oc_wrapper get pods --ignore-not-found=true -n $ns -o "jsonpath={range .items[*]}{' '}{.metadata.name}{':'}{range .spec['containers', 'initContainers'][*]}{.name}{','}"))
     for pc in ${PODS_CONTAINERS[@]}; do
         # https://stackoverflow.com/a/4444841
         POD=${pc%%:*}
         CONTAINERS=${pc#*:}
         for container in ${CONTAINERS//,/ }; do
-            oc logs $POD -c $container -n $ns > $LOGS_DIR/${POD}_${container}.log 2> /dev/null || continue
-            oc logs $POD -c $container --previous -n $ns > $LOGS_DIR/${POD}_${container}-previous.log 2> /dev/null || continue
+            oc_wrapper logs $POD -c $container -n $ns > $LOGS_DIR/${POD}_${container}.log 2> /dev/null || continue
+            oc_wrapper logs $POD -c $container --previous -n $ns > $LOGS_DIR/${POD}_${container}-previous.log 2> /dev/null || continue
         done
     done
 }
@@ -47,11 +47,11 @@ function collect_k8s_artifacts() {
     mkdir -p $DIR
     get_pod_logs $ns
     echo "Collecting events and k8s configs..."
-    oc get events -n $ns --sort-by='.lastTimestamp' > $DIR/oc_get_events.txt
-    oc get all -n $ns -o yaml > $DIR/oc_get_all.yaml
-    oc get clowdapp -n $ns -o yaml > $DIR/oc_get_clowdapp.yaml
-    oc get clowdenvironment env-$ns -o yaml > $DIR/oc_get_clowdenvironment.yaml
-    oc get clowdjobinvocation -n $ns -o yaml > $DIR/oc_get_clowdjobinvocation.yaml
+    oc_wrapper get events -n $ns --sort-by='.lastTimestamp' > $DIR/oc_get_events.txt
+    oc_wrapper get all -n $ns -o yaml > $DIR/oc_get_all.yaml
+    oc_wrapper get clowdapp -n $ns -o yaml > $DIR/oc_get_clowdapp.yaml
+    oc_wrapper get clowdenvironment env-$ns -o yaml > $DIR/oc_get_clowdenvironment.yaml
+    oc_wrapper get clowdjobinvocation -n $ns -o yaml > $DIR/oc_get_clowdjobinvocation.yaml
 }
 
 function teardown {

cicd/bin/oc_wrapper

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+
+import sys
+import logging
+
+from sh import ErrorReturnCode
+from bonfire.openshift import oc
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    args = sys.argv[1:]
+    try:
+        oc(*args, _silent=True, _print=True)
+    except ErrorReturnCode as err:
+        sys.exit(err.exit_code)

cicd/bootstrap.sh

@@ -63,48 +63,14 @@ python3 -m venv .bonfire_venv
 source .bonfire_venv/bin/activate
 
 pip install --upgrade pip 'setuptools<58' wheel
-pip install --upgrade crc-bonfire
+pip install --upgrade 'crc-bonfire>=3.6.0'
 
 # clone repo to download cicd scripts
 rm -fr $BONFIRE_ROOT
 git clone --branch master https://github.com/RedHatInsights/bonfire.git $BONFIRE_ROOT
 
-# Add a retry mechanism to 'oc' command calls
-oc_wrapper() {
-  # hide all the extra stuff we're doing in here if user called 'set -x' before 'oc'
-  # to make debugging log output easier
-  # https://stackoverflow.com/a/50668339
-  if [ -o xtrace ]; then
-      set +x
-      trap 'set -x' RETURN
-  fi
-
-  real_oc=$(which oc)
-  retries=3
-  backoff=3
-  attempt=0
-
-  if [ -z "$real_oc" ]; then
-    echo "ERROR: unable to locate 'oc' command on PATH" >&2
-    return 1
-  fi
-
-  while true; do
-    attempt=$((attempt+1))
-    $real_oc $@ && return 0  # exit here if 'oc' completes successfully
-
-    if [ "$attempt" -lt $retries ]; then
-      sleep_time=$(($attempt*$backoff))
-      echo "oc command hit error (attempt $attempt/$retries), retrying in $sleep_time sec" >&2
-      sleep $sleep_time
-    else
-      break
-    fi
-  done
-
-  echo "oc command failed, gave up after $retries tries" >&2
-  return 1
-}
+# Gives access to helper commands such as "oc_wrapper"
+export PATH=$PATH:${CICD_ROOT}/bin
 
 # log in to ephemeral cluster
 oc_wrapper login --token=$OC_LOGIN_TOKEN --server=$OC_LOGIN_SERVER

cicd/cji_smoke_test.sh

@@ -57,7 +57,7 @@ oc_wrapper logs -n $NAMESPACE $POD -f &
 # Wait for the job to Complete or Fail before we try to grab artifacts
 # condition=complete does trigger when the job fails
 set -x
-oc wait --timeout=$IQE_CJI_TIMEOUT --for=condition=JobInvocationComplete -n $NAMESPACE cji/$CJI_NAME
+oc_wrapper wait --timeout=$IQE_CJI_TIMEOUT --for=condition=JobInvocationComplete -n $NAMESPACE cji/$CJI_NAME
 set +x
 
 # Log in again -- perhaps this will fix the intermittent 'service account lost permissions' error?