Add logic to retry mc artifact copy (#151)

* Add logic to retry mc artifact copy

cicd/_common_deploy_logic.sh

@@ -66,6 +66,9 @@ function collect_k8s_artifacts {
 
 function teardown {
     [ "$TEARDOWN_RAN" -ne "0" ] && return
+    echo "------------------------"
+    echo "----- TEARING DOWN -----"
+    echo "------------------------"
     if [ ! -z "$NAMESPACE" ]; then
         set +e
         collect_k8s_artifacts

cicd/cji_smoke_test.sh

@@ -16,6 +16,8 @@
 # In order for the deploy-iqe-cji to run correctly, we must set the marker and filter to "" if they
 # are not already set by caller
 # https://unix.stackexchange.com/questions/122845/using-a-b-for-variable-assignment-in-scripts/122848#122848
+set -e
+
 : "${IQE_MARKER_EXPRESSION:='""'}"
 : "${IQE_FILTER_EXPRESSION:='""'}"
 : "${IQE_IMAGE_TAG:='""'}"
@@ -82,11 +84,36 @@ CMD="mkdir -p /artifacts &&
 mc --no-color --quiet alias set minio http://${MINIO_HOST}:${MINIO_PORT} ${MINIO_ACCESS} ${MINIO_SECRET_KEY} &&
 mc --no-color --quiet mirror --overwrite minio/${BUCKET_NAME} /artifacts/
 "
-set -x
-docker run -t --net=host --name=$CONTAINER_NAME --entrypoint="/bin/sh" $MC_IMAGE -c "$CMD"
-docker cp $CONTAINER_NAME:/artifacts/. $ARTIFACTS_DIR
-docker rm $CONTAINER_NAME
-set +x
+
+run_mc () {
+    echo "running: docker run -t --net=host --name=$CONTAINER_NAME --entrypoint=\"/bin/sh\" $MC_IMAGE -c \"$CMD\""
+    set +e
+    docker run -t --net=host --name=$CONTAINER_NAME --entrypoint="/bin/sh" $MC_IMAGE -c "$CMD"
+    RET_CODE=$?
+    docker cp $CONTAINER_NAME:/artifacts/. $ARTIFACTS_DIR
+    docker rm $CONTAINER_NAME
+    set -e
+    return $RET_CODE
+}
+
+# Add retry logic for intermittent minio connection failures
+MINIO_SUCCESS=false
+for i in $(seq 1 5); do
+    if run_mc; then
+        MINIO_SUCCESS=true
+        break
+    else
+        if [ "$i" -lt "5" ]; then
+            echo "WARNING: minio artifact copy failed, retrying in 5sec..."
+            sleep 5
+        fi
+    fi
+done
+
+if [ "$MINIO_SUCCESS" = false ]; then
+    echo "ERROR: minio artifact copy failed"
+    exit 1
+fi
 
 echo "copied artifacts from iqe pod: "
 ls -l $ARTIFACTS_DIR