From 1c69838bdc830035d7403a520d9d8d6492c2c28d Mon Sep 17 00:00:00 2001
From: Yifeng Xiao <yifengx@vmware.com>
Date: Fri, 24 Aug 2018 10:38:31 +0800
Subject: [PATCH] Initial Jenkins scenario jobs (#8)

Add jenkins job configurations and scripts for vic and
vic-product scenario tests.
---
 jenkins/jjb/projects.yaml                     |  55 +++++++
 jenkins/jjb/templates/scenario-tests.yaml     | 149 +++++++++++++++++
 jenkins/jjb/templates/scenario-trigger.yaml   |  15 ++
 jenkins/jjb/views.yaml                        |  17 ++
 .../build.sh}                                 |  24 ++-
 .../jenkins_job_trigger.py                    |  13 +-
 jenkins/jobs/vic-product-scenario/build.sh    | 102 ++++++++++++
 .../vic-product-scenario/selenium_grid.sh     |  76 +++++++++
 jenkins/jobs/vic-scenario/build.sh            | 150 ++++++++++++++++++
 jenkins/jobs/vic-scenario/upload-logs.sh      |  71 +++++++++
 10 files changed, 660 insertions(+), 12 deletions(-)
 create mode 100644 jenkins/jjb/projects.yaml
 create mode 100644 jenkins/jjb/templates/scenario-tests.yaml
 create mode 100644 jenkins/jjb/templates/scenario-trigger.yaml
 create mode 100644 jenkins/jjb/views.yaml
 rename jenkins/jobs/{trigger/nightly_trigger.sh => scenario-trigger/build.sh} (73%)
 rename jenkins/jobs/{trigger => scenario-trigger}/jenkins_job_trigger.py (86%)
 create mode 100755 jenkins/jobs/vic-product-scenario/build.sh
 create mode 100755 jenkins/jobs/vic-product-scenario/selenium_grid.sh
 create mode 100755 jenkins/jobs/vic-scenario/build.sh
 create mode 100755 jenkins/jobs/vic-scenario/upload-logs.sh

diff --git a/jenkins/jjb/projects.yaml b/jenkins/jjb/projects.yaml
new file mode 100644
index 0000000..ea963d4
--- /dev/null
+++ b/jenkins/jjb/projects.yaml
@@ -0,0 +1,55 @@
+- project:
+    name: 'scenario-tests-trigger'
+    jobs:
+      - '{repo}-{version}-trigger':
+          repo: 'vic'
+          version: 'master'
+          branch: 'master'
+          prefix: 'vic_'
+          bucket: 'vic-engine-builds'
+      - '{repo}-{version}-trigger':
+          repo: 'vic'
+          version: '1.4.3'
+          branch: 'releases/1.4.3'
+          prefix: 'vic_'
+          bucket: 'vic-engine-builds'
+      - '{repo}-{version}-trigger':
+          repo: 'vic-product'
+          version: 'master'
+          branch: 'master'
+          prefix: 'vic-dev'
+          bucket: 'vic-product-ova-builds'
+      - '{repo}-{version}-trigger':
+          repo: 'vic-product'
+          version: '1.4.3'
+          branch: 'releases/1.4.3'
+          prefix: 'vic-dev'
+          bucket: 'vic-product-ova-builds'
+
+- project:
+    name: 'scenario-tests'
+    default_esx_build: '8169922'
+    default_vc_build: '8217866'
+    build_timeout: '1440'
+    parallel_jobs: '1'
+    jobs:
+      - 'vic-{version}-scenario':
+          repo: 'vic'
+          version: '1.4.3'
+          branch: 'releases/1.4.3'
+          node: 'vic-executor'
+      - 'vic-{version}-scenario':
+          repo: 'vic'
+          version: 'master'
+          branch: 'master'
+          node: 'vic-executor'
+      - 'vic-product-{version}-scenario':
+          repo: 'vic-product'
+          version: '1.4.3'
+          branch: 'releases/1.4.3'
+          node: 'vic-executor1'
+      - 'vic-product-{version}-scenario':
+          repo: 'vic-product'
+          version: 'master'
+          branch: 'master'
+          node: 'vic-executor1'
\ No newline at end of file
diff --git a/jenkins/jjb/templates/scenario-tests.yaml b/jenkins/jjb/templates/scenario-tests.yaml
new file mode 100644
index 0000000..2d7efcc
--- /dev/null
+++ b/jenkins/jjb/templates/scenario-tests.yaml
@@ -0,0 +1,149 @@
+- defaults:
+    name: 'scenario-tests'
+    description: 'This is a parameterized build for running Nimbus scenario tests against build of {repo} {branch}'
+    node: '{node}'
+    parameters:
+      - choice:
+          name: VSPHERE_VERSION
+          choices:
+            - '6.7'
+            - '6.5'
+            - '6.0'
+          description: 'The version of vSphere to run against. Skipped tests are determined by it.'
+      - string:
+          name: ESX_BUILD
+          default: '{default_esx_build}'
+          description: 'The build of ESX to deploy, e.g. ob-8169922 (6.7 default).'
+      - string:
+          name: VC_BUILD
+          default: '{default_vc_build}'
+          description: 'The build of VC to deploy, e.g. ob-8217866 (6.7 default).'
+      - string:
+          name: BUILD_NUM
+          default: ''
+          description: 'Build number to test.'
+      - string:
+          name: TEST_CASES
+          default: ''
+          description: |
+            The specific test cases to run as part of the job. These are passed as arguments to pabot so will likely be of the form "tests/manual-test-cases/Group23-Future-Tests"
+            Runs the default set of tests if left empty.
+    scm:
+      - git:
+          url: 'git@gitlab.eng.vmware.com:core-build/vic-internal.git'
+          credentials-id: 'GitLabAutomationKey'
+          branches:
+            - 'master'
+          basedir: 'vic-internal'
+      - git:
+          url: 'https://github.com/vmware/{repo}'
+          branches:
+            - '{branch}'
+          basedir: '{repo}'
+      - git:
+          url: 'https://github.com/vmware/vic-tools'
+          branches:
+            - 'master'
+          basedir: 'vic-tools'
+    wrappers:
+      - timeout:
+          name: absolute_timeout
+          timeout: '{build_timeout}'
+          abort: true
+          type: absolute
+      - workspace-cleanup
+      - build-name:
+          name: ${{ENV,var="BUILD_NUM"}}
+    publishers:
+      - robot:
+          output-path: '{repo}/report'
+          pass-threshold: '100'
+          unstable-threshold: '100'
+      - groovy-postbuild:
+          script: |
+              def vc_build = manager.envVars['VC_BUILD']
+              def esx_build = manager.envVars['ESX_BUILD']
+              def vsphere_version = manager.envVars.get('VSPHERE_VERSION')
+              def summary = manager.createSummary("gear2.gif")
+              summary.appendText("Infrastructure Builds:<ul>", false)
+              summary.appendText("<li><b>vSphere Version</b> - ${{vsphere_version}}</li>", false)
+              summary.appendText("<li><b>vCenter</b> - ${{vc_build}}</li>", false)
+              summary.appendText("<li><b>ESXi</b> - ${{esx_build}}</li>", false)
+      - email-ext:
+          recipients: vic-eng@vmware.com
+          reply-to: vic-eng@vmware.com
+          success: true
+          aborted: true
+          failure: true
+          content-type: html
+          send-to:
+            - recipients
+          subject: 'vmware/{repo} Scenario Testing - ${{VSPHERE_VERSION}}/{repo}/{branch} - Build: $BUILD_NUM'
+          body: |
+              Download link for the test logs is in the <a href="${{BUILD_URL}}/console">build console output</a>
+              <p>
+              ${{VSPHERE_VERSION}} Jenkins Nimbus Scenario Job: <a href="${{BUILD_URL}}">${{BUILD_URL}}</a>
+              <p>
+              Run Report: ${{ROBOT_REPORTLINK}}
+              <p>
+              ${{FILE,path="{repo}/console.log"}}
+      - slack:
+          room: '#notifications'
+          auth-token-credential-id: 'SlackIntegrationToken'
+          notify-success: True
+          notify-aborted: True
+          notify-failure: True
+          base-url: 'https://vmware-vic.slack.com/services/hooks/jenkins-ci/'
+          include-custom-message: True
+          custom-message: 'vmware/{repo} Scenario Testing - ${{VSPHERE_VERSION}}/{repo}/{branch} - $BUILD_TIMESTAMP'
+
+- job-template:
+    name: 'vic-{version}-scenario'
+    defaults: 'scenario-tests'
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -x
+          dest="vic-ci-logs/vmware/{branch}/"
+
+          cat > build.envfile <<ENVS
+          PARALLEL_JOBS={parallel_jobs}
+          NIMBUS_RETRY_ATTEMPTS=5
+          NIMBUS_RETRY_DELAY=1m
+          LOG_UPLOAD_DEST=$dest
+          GIT_COMMIT=$GIT_COMMIT
+          BUILD_TIMESTAMP=$BUILD_TIMESTAMP
+          BUILD_ID=$BUILD_ID
+          ESX_BUILD=$ESX_BUILD
+          VC_BUILD=$VC_BUILD
+          BUILD_TAG={repo}_${{BUILD_NUM}}
+          VCH_BUILD=${{BUILD_NUM}}
+          VCH_BRANCH={branch}
+          ENVS
+
+          docker run --rm -v "${{WORKSPACE}}":/go --env-file vic-internal/{repo}-scenario-{version}-secrets.list --env-file build.envfile --name=${{BUILD_TAG}} gcr.io/eminent-nation-87317/vic-integration-test:1.48 vic-tools/jenkins/jobs/vic-scenario/build.sh ${{VSPHERE_VERSION}} ${{TEST_CASES}}
+
+- job-template:
+    name: 'vic-product-{version}-scenario'
+    defaults: 'scenario-tests'
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -x
+
+          ./vic-tools/jenkins/jobs/vic-product-scenario/selenium_grid.sh create {version}
+
+          cat > build.envfile <<ENVS
+          PARALLEL_JOBS={parallel_jobs}
+          ROBOT_REPORT=report
+          NIMBUS_RETRY_ATTEMPTS=5
+          NIMBUS_RETRY_DELAY=1m
+          ESX_BUILD=$ESX_BUILD
+          VC_BUILD=$VC_BUILD
+          BUILD_TAG={repo}_${{BUILD_NUM}}
+          VIC_PRODUCT_BUILD=${{BUILD_NUM}}
+          VIC_PRODUCT_BRANCH={branch}
+          ENVS
+
+          docker run --net grid-{version} --privileged --rm --link selenium-hub-{version}:selenium-grid-hub -v /var/run/docker.sock:/var/run/docker.sock -v /etc/docker/certs.d:/etc/docker/certs.d -v "${{WORKSPACE}}":/go -v /vic-cache:/vic-cache --env-file "vic-internal/{repo}-scenario-{version}-secrets.list" --env-file build.envfile gcr.io/eminent-nation-87317/vic-integration-test:1.46 vic-tools/jenkins/jobs/vic-product-scenario/build.sh ${{TEST_CASES}}
+          ./vic-tools/jenkins/jobs/vic-product-scenario/selenium_grid.sh remove {version}
\ No newline at end of file
diff --git a/jenkins/jjb/templates/scenario-trigger.yaml b/jenkins/jjb/templates/scenario-trigger.yaml
new file mode 100644
index 0000000..f60b224
--- /dev/null
+++ b/jenkins/jjb/templates/scenario-trigger.yaml
@@ -0,0 +1,15 @@
+- job-template:
+    name: '{repo}-{version}-trigger'
+    description: 'This is a parameterized build for running Nimbus scenario tests against build of {repo} {branch}'
+    node: 'vic-executor'
+    scm:
+      - git:
+          url: 'https://github.com/vmware/vic-tools'
+          branches:
+            - 'master'
+    triggers:
+      - timed: '@midnight'
+    builders:
+      - shell: |
+          #!/bin/bash
+          docker run  --rm -v ${{WORKSPACE}}:/src wdc-harbor-ci.eng.vmware.com/default-project/ci-trigger:0.0.1 /src/jenkins/jobs/scenario-trigger/build.sh {bucket} {branch} {prefix} {repo}-{version}-scenario
diff --git a/jenkins/jjb/views.yaml b/jenkins/jjb/views.yaml
new file mode 100644
index 0000000..c0fd0a0
--- /dev/null
+++ b/jenkins/jjb/views.yaml
@@ -0,0 +1,17 @@
+- view:
+    name: 'Scenario'
+    view-type: list
+    description: 'VIC Scenario Tests runs nightly'
+    job-name:
+      - vic-1.4.3-scenario
+      - vic-master-scenario
+      - vic-product-1.4.3-scenario
+      - vic-product-master-scenario
+    columns:
+      - status
+      - weather
+      - job
+      - last-success
+      - last-failure
+      - last-duration
+      - robot-list
diff --git a/jenkins/jobs/trigger/nightly_trigger.sh b/jenkins/jobs/scenario-trigger/build.sh
similarity index 73%
rename from jenkins/jobs/trigger/nightly_trigger.sh
rename to jenkins/jobs/scenario-trigger/build.sh
index bd901ae..da7bd50 100755
--- a/jenkins/jobs/trigger/nightly_trigger.sh
+++ b/jenkins/jobs/scenario-trigger/build.sh
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Usage example: ./nightly_trigger.sh vic-engine-builds master vic_ vic-master-nightly
+# Usage example: ./build.sh vic-engine-builds master vic_ vic-master-nightly
 
 set -ex
 
@@ -28,16 +28,26 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 # Get the latest build filename
 if [ "${REPO_BRANCH}" == "master" ]; then
-    GS_PATH="${ARTIFACT_BUCKET}/"
+    GS_PATH="${ARTIFACT_BUCKET}"
 else
-    GS_PATH="${ARTIFACT_BUCKET}/${REPO_BRANCH}/"
+    GS_PATH="${ARTIFACT_BUCKET}/${REPO_BRANCH}"
 fi
-FILE_NAME=$(gsutil ls -l "gs://${GS_PATH}${BINARY_PREFIX}*" | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | xargs basename)
+FILE_NAME=$(gsutil ls -l "gs://${GS_PATH}/${BINARY_PREFIX}*" | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | xargs basename)
 
 # strip prefix and suffix from archive filename
-BUILD_NUM=${FILE_NAME#${BINARY_PREFIX}}
-BUILD_NUM=${BUILD_NUM%%.*}
-
+case ${ARTIFACT_BUCKET} in
+    vic-engine-builds)
+        BUILD_NUM=${FILE_NAME#${BINARY_PREFIX}}
+        BUILD_NUM=${BUILD_NUM%%.*}
+        ;;
+    "vic-product-ova-builds")
+        BUILD_NUM=$(echo ${FILE_NAME} | awk -F '-' '{NF--;  print $NF }')
+        ;;
+    *)
+        echo "Bucket ${ARTIFACT_BUCKET} is not supported."
+        exit 1
+        ;;
+esac
 echo "Trigger build ${BUILD_NUM}"
 
 # Run test on vsphere 6.0, 6.5, 6.7 alternatively
diff --git a/jenkins/jobs/trigger/jenkins_job_trigger.py b/jenkins/jobs/scenario-trigger/jenkins_job_trigger.py
similarity index 86%
rename from jenkins/jobs/trigger/jenkins_job_trigger.py
rename to jenkins/jobs/scenario-trigger/jenkins_job_trigger.py
index 4285a4b..9c7c7fc 100644
--- a/jenkins/jobs/trigger/jenkins_job_trigger.py
+++ b/jenkins/jobs/scenario-trigger/jenkins_job_trigger.py
@@ -5,12 +5,12 @@
 """
 
 import logging
-
 import argparse
+
 from jenkinsapi.custom_exceptions import NoBuildData
 from jenkinsapi.jenkins import Jenkins
 from jenkinsapi.api import log as jenkins_api_log
-
+from jenkinsapi.constants import STATUS_SUCCESS
 
 formatter = logging.Formatter('%(asctime)s | %(message)s', '%Y-%m-%d %H:%M:%S')
 console_handler = logging.StreamHandler()
@@ -24,9 +24,9 @@
 jenkins_api_log.addHandler(console_handler)
 
 
-def is_tested(jenkinsci, job_name, build_num):
+def is_verified(jenkinsci, job_name, build_num):
     """
-    Checks to see whether a specified build has already been tested.
+    Checks to see whether a specified build has already been verified.
     """
     try:
         job = jenkinsci[job_name]
@@ -43,6 +43,9 @@ def is_tested(jenkinsci, job_name, build_num):
         return False
 
     LOG.info('Build %s is already tested', build_num)
+    if build._data['result'] != STATUS_SUCCESS:
+        LOG.info('Build %s failed last time, rerun it', build_num)
+        return False
     return True
 
 
@@ -62,7 +65,7 @@ def main():
     args = parser.parse_args()
     jenkinsci = Jenkins(args.jenkins, username=args.username,
                         password=args.password)
-    if not is_tested(jenkinsci, args.job_name, args.build_num):
+    if not is_verified(jenkinsci, args.job_name, args.build_num):
         params = {'VSPHERE_VERSION': args.vsphere_version,
                   'ESX_BUILD': args.esx_build,
                   'VC_BUILD': args.vc_build,
diff --git a/jenkins/jobs/vic-product-scenario/build.sh b/jenkins/jobs/vic-product-scenario/build.sh
new file mode 100755
index 0000000..262fe62
--- /dev/null
+++ b/jenkins/jobs/vic-product-scenario/build.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+# Copyright 2018 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+set -x
+
+WORKSPACE_DIR=$(cd $(dirname "$0")/../../../.. && pwd)
+
+# 6.0u3
+ESX_60_VERSION="ob-5050593"
+VC_60_VERSION="ob-5112509" # the cloudvm build corresponding to the vpx build
+
+# 6.5u2
+ESX_65_VERSION="ob-8935087"
+VC_65_VERSION="ob-8307201"
+
+# 6.7
+ESX_67_VERSION="ob-8169922"
+VC_67_VERSION="ob-8217866"
+
+
+DEFAULT_TESTCASES=("tests/manual-test-cases")
+
+DEFAULT_VIC_PRODUCT_BRANCH=""
+DEFAULT_VIC_PRODUCT_BUILD="*"
+
+DEFAULT_PARALLEL_JOBS=4
+
+echo "Target version: ${VSPHERE_VERSION}"
+excludes=(--exclude skip)
+case "$VSPHERE_VERSION" in
+    "6.0")
+        excludes+=(--exclude nsx)
+        ESX_BUILD=${ESX_BUILD:-$ESX_60_VERSION}
+        VC_BUILD=${VC_BUILD:-$VC_60_VERSION}
+        ;;
+    "6.5")
+        ESX_BUILD=${ESX_BUILD:-$ESX_65_VERSION}
+        VC_BUILD=${VC_BUILD:-$VC_65_VERSION}
+        ;;
+    "6.7")
+        excludes+=(--exclude nsx --exclude hetero)
+        ESX_BUILD=${ESX_BUILD:-$ESX_67_VERSION}
+        VC_BUILD=${VC_BUILD:-$VC_67_VERSION}
+        ;;
+esac
+
+testcases=("${@:-${DEFAULT_TESTCASES[@]}}")
+${ARTIFACT_PREFIX:="vic-*"}
+${GCS_BUCKET:="vic-product-ova-builds"}
+
+VIC_PRODUCT_BRANCH=${VIC_PRODUCT_BRANCH:-${DEFAULT_VIC_PRODUCT_BRANCH}}
+VIC_PRODUCT_BUILD=${VIC_PRODUCT_BUILD:-${DEFAULT_VIC_PRODUCT_BUILD}}
+if [ "${VIC_PRODUCT_BRANCH}" == "master" ]; then
+    GS_PATH="${GCS_BUCKET}"
+else
+    GS_PATH="${GCS_BUCKET}/${VIC_PRODUCT_BRANCH}"
+fi
+input=$(gsutil ls -l "gs://${GS_PATH}/${ARTIFACT_PREFIX}-${VIC_PRODUCT_BUILD}-*" | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | xargs basename)
+constructed_url="https://storage.googleapis.com/${GS_PATH}/${input}"
+ARTIFACT_URL="${ARTIFACT_URL:-${constructed_url}}"
+input=$(basename "${ARTIFACT_URL}")
+
+pushd ${WORKSPACE_DIR}/vic-product
+    echo "Downloading VIC Product OVA build $input... from ${ARTIFACT_URL}"
+    n=0 && rm -f "${input}"
+    until [[ $n -ge 5 ]]; do
+        echo "Retry.. $n"
+        echo "Downloading gcp file ${input} from ${ARTIFACT_URL}"
+        wget --unlink -nv -O "${input}" "${ARTIFACT_URL}" && break;
+        # clean up any residual file from failed download
+        rm -f "${input}"
+        ((n++))
+        sleep 10;
+    done
+
+    if [[ ! -f $input ]]; then
+        echo "VIC Product OVA download failed"
+        exit 1
+    fi
+    echo "VIC Product OVA download complete..."
+
+    PARALLEL_JOBS=${PARALLEL_JOBS:-${DEFAULT_PARALLEL_JOBS}}
+    pabot --verbose --processes "${PARALLEL_JOBS}" -d report --removekeywords TAG:secret "${excludes[@]}" --variable ESX_VERSION:"${ESX_BUILD}" --variable VC_VERSION:"${VC_BUILD}" "${testcases[@]}"
+    cat report/pabot_results/*/stdout.txt | grep -E '::|\.\.\.' | grep -E 'PASS|FAIL' > console.log
+
+    # Pretty up the email results
+    sed -i -e 's/^/<br>/g' console.log
+    sed -i -e 's|PASS|<font color="green">PASS</font>|g' console.log
+    sed -i -e 's|FAIL|<font color="red">FAIL</font>|g' console.log
+popd
+
diff --git a/jenkins/jobs/vic-product-scenario/selenium_grid.sh b/jenkins/jobs/vic-product-scenario/selenium_grid.sh
new file mode 100755
index 0000000..209a151
--- /dev/null
+++ b/jenkins/jobs/vic-product-scenario/selenium_grid.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+# Copyright 2018 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+set -x
+
+start_node () {
+    docker run -d --net ${NETWORK} -e HUB_HOST=${HUB_NM} -v /dev/shm:/dev/shm --name "$1" "$2"
+
+    for _ in $(seq 1 10); do
+        if [[ "$(docker logs "$1")" = *"The node is registered to the hub and ready to use"* ]]; then
+            echo "$1 node is up and ready to use";
+            return 0;
+        fi
+        sleep 3;
+    done
+}
+
+remove_all () {
+    echo "Kill any old selenium infrastructure..."
+    docker rm -f ${HUB_NM} ${NODE1_NM} ${NODE2_NM} ${NODE3_NM} ${NODE4_NM}
+    docker network prune -f
+}
+
+if [ $# -ne 2 ] ; then
+    echo "Usage: $0 create/remove suffix"
+    exit 1
+fi
+
+ACTION=$1
+SUFFIX=$2
+NETWORK="grid-${SUFFIX}"
+HUB_NM="selenium-hub-${SUFFIX}"
+NODE1_NM="firefox1-${SUFFIX}"
+NODE2_NM="firefox2-${SUFFIX}"
+NODE3_NM="firefox3-${SUFFIX}"
+NODE4_NM="firefox4-${SUFFIX}"
+
+case ${ACTION} in
+    create)
+        remove_all
+        echo "Create the network, hub and workers..."
+        docker network create ${NETWORK}
+        docker run -d --net ${NETWORK} --name ${HUB_NM} selenium/hub:3.9.1
+        for _ in $(seq 1 10); do
+            if [[ "$(docker logs ${HUB_NM} 2>&1)" = *"Selenium Grid hub is up and running"* ]]; then
+                echo 'Selenium Server is up and running';
+                break
+            fi
+            sleep 3;
+        done
+
+        start_node ${NODE1_NM} selenium/node-firefox:3.9.1
+        start_node ${NODE2_NM} selenium/node-firefox:3.9.1
+        start_node ${NODE3_NM} selenium/node-firefox:3.9.1
+        start_node ${NODE4_NM} selenium/node-firefox:3.9.1
+        ;;
+    remove)
+        remove_all
+        ;;
+    *)
+        echo "${ACTION} is not supported."
+        exit 1
+        ;;
+esac
+
diff --git a/jenkins/jobs/vic-scenario/build.sh b/jenkins/jobs/vic-scenario/build.sh
new file mode 100755
index 0000000..99cb13d
--- /dev/null
+++ b/jenkins/jobs/vic-scenario/build.sh
@@ -0,0 +1,150 @@
+#!/bin/bash
+# Copyright 2018 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Permalink to ESX build numbers/versions: https://kb.vmware.com/kb/2143832
+# Permalink to vCenter build numbers/versions: https://kb.vmware.com/kb/2143838
+set -x
+
+# 6.0u3
+ESX_60_VERSION="ob-5050593"
+VC_60_VERSION="ob-5112509" # the cloudvm build corresponding to the vpx build that is noted in the kb link above
+
+# 6.5u2
+ESX_65_VERSION="ob-8935087"
+VC_65_VERSION="ob-8307201"
+
+# 6.7
+ESX_67_VERSION="ob-8169922"
+VC_67_VERSION="ob-8217866"
+
+DEFAULT_LOG_UPLOAD_DEST="vic-ci-logs"
+DEFAULT_ARTIFACT_BUCKET="vic-engine-builds"
+DEFAULT_VCH_BRANCH=""
+DEFAULT_VCH_BUILD="*"
+DEFAULT_TESTCASES=("tests/manual-test-cases/Group5-Functional-Tests" "tests/manual-test-cases/Group13-vMotion" "tests/manual-test-cases/Group21-Registries" "tests/manual-test-cases/Group23-Future-Tests")
+
+DEFAULT_PARALLEL_JOBS=4
+DEFAULT_RUN_AS_OPS_USER=0
+
+VIC_BINARY_PREFIX="vic_"
+
+# This is exported to propagate into the pybot processes launched by pabot
+export RUN_AS_OPS_USER=${RUN_AS_OPS_USER:-${DEFAULT_RUN_AS_OPS_USER}}
+
+PARALLEL_JOBS=${PARALLEL_JOBS:-${DEFAULT_PARALLEL_JOBS}}
+
+SCRIPT_DIR=$(cd $(dirname "$0") && pwd)
+
+if [[ $1 != "6.0" && $1 != "6.5" && $1 != "6.7" ]]; then
+    echo "Please specify a target version. One of: 6.0, 6.5, 6.7"
+    exit 1
+fi
+
+# process the CLI arguments
+target="$1"
+echo "Target version: ${target}"
+shift
+# Take the remaining CLI arguments as a test case list - this is treated as an array to preserve quoting when passing to pabot
+testcases=("${@:-${DEFAULT_TESTCASES[@]}}")
+
+# TODO: the version downloaded by this logic is not coupled with the tests that will be run against it. This should be altered to pull a version that matches the commit SHA of the tests
+# we will be running or similar mechanism.
+VCH_BUILD=${VCH_BUILD:-${DEFAULT_VCH_BUILD}}
+VCH_BRANCH=${VCH_BRANCH:-${DEFAULT_VCH_BRANCH}}
+ARTIFACT_BUCKET=${ARTIFACT_BUCKET:-${DEFAULT_ARTIFACT_BUCKET}}
+if [ "${VCH_BRANCH}" == "master" ]; then
+    GS_PATH="${ARTIFACT_BUCKET}"
+else
+    GS_PATH="${ARTIFACT_BUCKET}/${VCH_BRANCH}"
+fi
+input=$(gsutil ls -l gs://${GS_PATH}/${VIC_BINARY_PREFIX}${VCH_BUILD}.tar.gz | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | xargs basename)
+constructed_url="https://storage.googleapis.com/${GS_PATH}/${input}"
+ARTIFACT_URL="${ARTIFACT_URL:-${constructed_url}}"
+input=$(basename ${ARTIFACT_URL})
+
+# strip prefix and suffix from archive filename
+VCH_BUILD=${input#${VIC_BINARY_PREFIX}}
+VCH_BUILD=${VCH_BUILD%%.*}
+
+# Enforce short SHA
+GIT_COMMIT=${GIT_COMMIT:0:7}
+
+case "$target" in
+    "6.0")
+        excludes="--exclude nsx"
+        ESX_BUILD=${ESX_BUILD:-$ESX_60_VERSION}
+        VC_BUILD=${VC_BUILD:-$VC_60_VERSION}
+        ;;
+    "6.5")
+        ESX_BUILD=${ESX_BUILD:-$ESX_65_VERSION}
+        VC_BUILD=${VC_BUILD:-$VC_65_VERSION}
+        ;;
+    "6.7")
+        excludes="--exclude nsx --exclude hetero"
+        ESX_BUILD=${ESX_BUILD:-$ESX_67_VERSION}
+        VC_BUILD=${VC_BUILD:-$VC_67_VERSION}
+        ;;
+esac
+
+LOG_UPLOAD_DEST="${LOG_UPLOAD_DEST:-${DEFAULT_LOG_UPLOAD_DEST}}"
+
+pushd vic
+    n=0 && rm -f "${input}"
+    until [ $n -ge 5 -o -f "${input}" ]; do
+        echo "Retry.. $n"
+        echo "Downloading gcp file ${input} from ${ARTIFACT_URL}"
+        wget --unlink -nv -O "${input}" "${ARTIFACT_URL}" && break;
+        # clean up any residual file from failed download
+        rm -f "${input}"
+        ((n++))
+        sleep 15
+    done
+
+    echo "Extracting .tar.gz"
+    mkdir bin && tar xvzf ${input} -C bin/ --strip 1
+
+    if [ ! -f  "bin/vic-machine-linux" ]; then
+        echo "Tarball extraction failed..quitting the run"
+        rm -rf bin
+        exit
+    else
+        VCH_COMMIT=$(bin/vic-machine-linux version | awk -F '-' '{print $NF}')
+        echo "Tarball extraction passed, Running nightlies test.."
+    fi
+
+    pabot --processes ${PARALLEL_JOBS} --removekeywords TAG:secret ${excludes} --variable ESX_VERSION:"${ESX_BUILD}" --variable VC_VERSION:"${VC_BUILD}" -d report "${testcases[@]}"
+    cat report/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log
+
+    # See if any VMs leaked
+    # TODO: should be a warning until clean, then changed to a failure if any leak
+    echo "There should not be any VMs listed here"
+    echo "======================================="
+    timeout 60s sshpass -p ${NIMBUS_PASSWORD} ssh -o StrictHostKeyChecking\=no ${NIMBUS_USER}@${NIMBUS_GW} nimbus-ctl list
+    echo "======================================="
+    echo "If VMs are listed we should investigate why they are leaking"
+
+    # archive the logs
+    logarchive="logs_vch-${VCH_BUILD}-${VCH_COMMIT}_test-${BUILD_ID}-${GIT_COMMIT}_${BUILD_TIMESTAMP}.zip"
+    /usr/bin/zip -9 -r "${logarchive}" report *.zip *.log *.debug *.tgz
+    if [ $? -eq 0 ]; then
+        ${SCRIPT_DIR}/upload-logs.sh ${logarchive} ${LOG_UPLOAD_DEST}
+    fi
+
+    # Pretty up the email results
+    sed -i -e 's/^/<br>/g' console.log
+    sed -i -e 's|PASS|<font color="green">PASS</font>|g' console.log
+    sed -i -e 's|FAIL|<font color="red">FAIL</font>|g' console.log
+popd
\ No newline at end of file
diff --git a/jenkins/jobs/vic-scenario/upload-logs.sh b/jenkins/jobs/vic-scenario/upload-logs.sh
new file mode 100755
index 0000000..c1a1ca0
--- /dev/null
+++ b/jenkins/jobs/vic-scenario/upload-logs.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+# Copyright 2016-2018 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+source=${1:?A source file must be specified}
+# The target destination in cloud storage. If the destination ends with a / it will be treated as a directory.
+# If not it _may_ be treated as a directory or a file depending on current remote objects. see gsutil cp doc.
+dest=${2:?A bucket must be specified, eg vic-ci-logs, or vic-ci-logs/user/branch/}
+
+if [ ! -r "${source}" ]; then
+  echo "Specified source file does not exist or cannot be read: ${source}"
+  exit 1
+fi
+
+if [ ${dest:0:1} == "/" ]; then
+  echo "Destination must start with a bucket name and no leading /"
+  exit 1
+fi
+
+# extract first path element as bucket name
+bucket=${dest%%/*}
+# drop bucket portion from path
+path=${dest#*/}
+# drop trailing / if any
+path=${path%/}
+
+# GC credentials
+keyfile=~/${bucket}.key
+botofile=~/.boto
+if [ ! -f $keyfile ]; then
+    echo -en $GS_PRIVATE_KEY > $keyfile
+    chmod 400 $keyfile
+fi
+if [ ! -f $botofile ]; then
+    echo "[Credentials]" >> $botofile
+    echo "gs_service_key_file = $keyfile" >> $botofile
+    echo "gs_service_client_id = $GS_CLIENT_EMAIL" >> $botofile
+    echo "[GSUtil]" >> $botofile
+    echo "content_language = en" >> $botofile
+    echo "default_project_id = $GS_PROJECT_ID" >> $botofile
+fi
+
+echo "----------------------------------------------"
+echo "Uploading logs to ${dest}"
+echo "----------------------------------------------"
+if gsutil cp "${source}" "gs://${dest}"; then
+  url="https://console.cloud.google.com/m/cloudstorage/b/${bucket}/o/${path}/${source}?authuser=1"
+  echo "$url" > log-download.url
+  echo "Download test logs here:"
+  echo "$url"
+else
+  echo "Log upload faled. Dumping gsutil version logic"
+  set -x
+  gsutil version -l
+  set +x
+fi
+echo "----------------------------------------------"
+
+rm -f $keyfile