Merge pull request openshift-kni#10 from fedepaol/external_tests

Consume the performance operator tier 1 tests

external-tests/clone_repo.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+set -e
+
+if [ "$TESTS_REPO" == "" ]; then
+    echo "[ERROR]: No TESTS_REPO provided"
+    exit 1
+fi
+
+if [ "$TESTS_LOCATION" == "" ]; then
+    echo "[ERROR]: No TESTS_LOCATION provided"
+    exit 1
+fi
+
+# detect latest master hash of the target repo if we're not pinned to a specific commit hash
+if [ -z "$TESTS_TARGET_HASH" ]; then
+    echo "Using latest master branch commit for $TESTS_REPO"
+    TESTS_TARGET_HASH=$(git ls-remote "$TESTS_REPO" | grep refs/heads/master | cut -f 1)
+fi
+
+echo "$TESTS_REPO commit hash: $TESTS_TARGET_HASH"
+
+if ! [ -d "$TESTS_LOCATION" ]; then
+    DOWNLOAD_SRC=true
+elif ! [ "$(cat "$TESTS_LOCATION"/git-hash)" = "$TESTS_TARGET_HASH" ]; then
+    rm -rf TESTS_LOCATION
+    DOWNLOAD_SRC=true
+fi
+
+if [ $DOWNLOAD_SRC ]; then
+    echo "Cloning code from $TESTS_REPO using hash $TESTS_TARGET_HASH"
+    # shellcheck disable=SC2086
+    repo_name=$(basename $TESTS_REPO)
+    curl -L "$TESTS_REPO"/archive/"$TESTS_TARGET_HASH"/"$repo_name".tar.gz | tar xz "$repo_name"-"$TESTS_TARGET_HASH"
+    mv "$repo_name"-"$TESTS_TARGET_HASH" "$TESTS_LOCATION"
+else
+    echo "Using cached $TESTS_LOCATION"
+fi
+
+echo "$TESTS_TARGET_HASH" > "$TESTS_LOCATION"/git-hash
+

external-tests/performance/test.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+export TESTS_REPO=https://github.com/openshift-kni/performance-addon-operators
+export TESTS_LOCATION=/tmp/performance-operators
+
+external-tests/clone_repo.sh
+cd $TESTS_LOCATION
+ROLE_WORKER_RT=worker-cnf PERF_TEST_PROFILE=performance make functests-only

hack/run-functests.sh

@@ -8,6 +8,13 @@ fi
 
 GOPATH="${GOPATH:-~/go}"
 export PATH=$PATH:$GOPATH/bin
+export failed=false
+export failures=()
+
+if [ "$FEATURES" == "" ]; then
+	echo "[ERROR]: No FEATURES provided"
+	exit 1
+fi
 
 which ginkgo
 if [ $? -ne 0 ]; then
@@ -18,4 +25,30 @@ fi
 
 FOCUS=$(echo "$FEATURES" | tr ' ' '|') 
 echo "Focusing on $FOCUS"
-GOFLAGS=-mod=vendor ginkgo --focus=$FOCUS functests -- -junit /tmp/artifacts/unit_report.xml
+if ! GOFLAGS=-mod=vendor ginkgo --focus=$FOCUS functests -- -junit /tmp/artifacts/unit_report.xml; then
+  failed=true
+  failures+=( "Tier 2 tests for $FEATURES" )
+fi
+
+for feature in $FEATURES; do
+  test_entry_point=external-tests/${feature}/test.sh
+  if [[ ! -f $test_entry_point ]]; then
+    echo "[INFO] Feature '$feature' does not have external tests entry point"
+    continue
+  fi
+  echo "[INFO] Running external tests for $feature"
+  set +e
+  if ! $test_entry_point; then
+    failures+=( "Tier 1 tests for $feature" )
+    failed=true
+  fi
+  set -e
+done
+
+if $failed; then
+  echo "[WARN] Tests failed:"
+  for failure in "${failures[@]}"; do
+    echo "$failure"
+  done;
+  exit 1
+fi