Initial Jenkins scenario jobs (#8)

Add jenkins job configurations and scripts for vic and
vic-product scenario tests.

jenkins/jjb/projects.yaml

@@ -0,0 +1,55 @@
+- project:
+    name: 'scenario-tests-trigger'
+    jobs:
+      - '{repo}-{version}-trigger':
+          repo: 'vic'
+          version: 'master'
+          branch: 'master'
+          prefix: 'vic_'
+          bucket: 'vic-engine-builds'
+      - '{repo}-{version}-trigger':
+          repo: 'vic'
+          version: '1.4.3'
+          branch: 'releases/1.4.3'
+          prefix: 'vic_'
+          bucket: 'vic-engine-builds'
+      - '{repo}-{version}-trigger':
+          repo: 'vic-product'
+          version: 'master'
+          branch: 'master'
+          prefix: 'vic-dev'
+          bucket: 'vic-product-ova-builds'
+      - '{repo}-{version}-trigger':
+          repo: 'vic-product'
+          version: '1.4.3'
+          branch: 'releases/1.4.3'
+          prefix: 'vic-dev'
+          bucket: 'vic-product-ova-builds'
+
+- project:
+    name: 'scenario-tests'
+    default_esx_build: '8169922'
+    default_vc_build: '8217866'
+    build_timeout: '1440'
+    parallel_jobs: '1'
+    jobs:
+      - 'vic-{version}-scenario':
+          repo: 'vic'
+          version: '1.4.3'
+          branch: 'releases/1.4.3'
+          node: 'vic-executor'
+      - 'vic-{version}-scenario':
+          repo: 'vic'
+          version: 'master'
+          branch: 'master'
+          node: 'vic-executor'
+      - 'vic-product-{version}-scenario':
+          repo: 'vic-product'
+          version: '1.4.3'
+          branch: 'releases/1.4.3'
+          node: 'vic-executor1'
+      - 'vic-product-{version}-scenario':
+          repo: 'vic-product'
+          version: 'master'
+          branch: 'master'
+          node: 'vic-executor1'

jenkins/jjb/templates/scenario-tests.yaml

@@ -0,0 +1,149 @@
+- defaults:
+    name: 'scenario-tests'
+    description: 'This is a parameterized build for running Nimbus scenario tests against build of {repo} {branch}'
+    node: '{node}'
+    parameters:
+      - choice:
+          name: VSPHERE_VERSION
+          choices:
+            - '6.7'
+            - '6.5'
+            - '6.0'
+          description: 'The version of vSphere to run against. Skipped tests are determined by it.'
+      - string:
+          name: ESX_BUILD
+          default: '{default_esx_build}'
+          description: 'The build of ESX to deploy, e.g. ob-8169922 (6.7 default).'
+      - string:
+          name: VC_BUILD
+          default: '{default_vc_build}'
+          description: 'The build of VC to deploy, e.g. ob-8217866 (6.7 default).'
+      - string:
+          name: BUILD_NUM
+          default: ''
+          description: 'Build number to test.'
+      - string:
+          name: TEST_CASES
+          default: ''
+          description: |
+            The specific test cases to run as part of the job. These are passed as arguments to pabot so will likely be of the form "tests/manual-test-cases/Group23-Future-Tests"
+            Runs the default set of tests if left empty.
+    scm:
+      - git:
+          url: '[email protected]:core-build/vic-internal.git'
+          credentials-id: 'GitLabAutomationKey'
+          branches:
+            - 'master'
+          basedir: 'vic-internal'
+      - git:
+          url: 'https://github.com/vmware/{repo}'
+          branches:
+            - '{branch}'
+          basedir: '{repo}'
+      - git:
+          url: 'https://github.com/vmware/vic-tools'
+          branches:
+            - 'master'
+          basedir: 'vic-tools'
+    wrappers:
+      - timeout:
+          name: absolute_timeout
+          timeout: '{build_timeout}'
+          abort: true
+          type: absolute
+      - workspace-cleanup
+      - build-name:
+          name: ${{ENV,var="BUILD_NUM"}}
+    publishers:
+      - robot:
+          output-path: '{repo}/report'
+          pass-threshold: '100'
+          unstable-threshold: '100'
+      - groovy-postbuild:
+          script: |
+              def vc_build = manager.envVars['VC_BUILD']
+              def esx_build = manager.envVars['ESX_BUILD']
+              def vsphere_version = manager.envVars.get('VSPHERE_VERSION')
+              def summary = manager.createSummary("gear2.gif")
+              summary.appendText("Infrastructure Builds:<ul>", false)
+              summary.appendText("<li><b>vSphere Version</b> - ${{vsphere_version}}</li>", false)
+              summary.appendText("<li><b>vCenter</b> - ${{vc_build}}</li>", false)
+              summary.appendText("<li><b>ESXi</b> - ${{esx_build}}</li>", false)
+      - email-ext:
+          recipients: [email protected]
+          reply-to: [email protected]
+          success: true
+          aborted: true
+          failure: true
+          content-type: html
+          send-to:
+            - recipients
+          subject: 'vmware/{repo} Scenario Testing - ${{VSPHERE_VERSION}}/{repo}/{branch} - Build: $BUILD_NUM'
+          body: |
+              Download link for the test logs is in the <a href="${{BUILD_URL}}/console">build console output</a>
+              <p>
+              ${{VSPHERE_VERSION}} Jenkins Nimbus Scenario Job: <a href="${{BUILD_URL}}">${{BUILD_URL}}</a>
+              <p>
+              Run Report: ${{ROBOT_REPORTLINK}}
+              <p>
+              ${{FILE,path="{repo}/console.log"}}
+      - slack:
+          room: '#notifications'
+          auth-token-credential-id: 'SlackIntegrationToken'
+          notify-success: True
+          notify-aborted: True
+          notify-failure: True
+          base-url: 'https://vmware-vic.slack.com/services/hooks/jenkins-ci/'
+          include-custom-message: True
+          custom-message: 'vmware/{repo} Scenario Testing - ${{VSPHERE_VERSION}}/{repo}/{branch} - $BUILD_TIMESTAMP'
+
+- job-template:
+    name: 'vic-{version}-scenario'
+    defaults: 'scenario-tests'
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -x
+          dest="vic-ci-logs/vmware/{branch}/"
+
+          cat > build.envfile <<ENVS
+          PARALLEL_JOBS={parallel_jobs}
+          NIMBUS_RETRY_ATTEMPTS=5
+          NIMBUS_RETRY_DELAY=1m
+          LOG_UPLOAD_DEST=$dest
+          GIT_COMMIT=$GIT_COMMIT
+          BUILD_TIMESTAMP=$BUILD_TIMESTAMP
+          BUILD_ID=$BUILD_ID
+          ESX_BUILD=$ESX_BUILD
+          VC_BUILD=$VC_BUILD
+          BUILD_TAG={repo}_${{BUILD_NUM}}
+          VCH_BUILD=${{BUILD_NUM}}
+          VCH_BRANCH={branch}
+          ENVS
+
+          docker run --rm -v "${{WORKSPACE}}":/go --env-file vic-internal/{repo}-scenario-{version}-secrets.list --env-file build.envfile --name=${{BUILD_TAG}} gcr.io/eminent-nation-87317/vic-integration-test:1.48 vic-tools/jenkins/jobs/vic-scenario/build.sh ${{VSPHERE_VERSION}} ${{TEST_CASES}}
+
+- job-template:
+    name: 'vic-product-{version}-scenario'
+    defaults: 'scenario-tests'
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -x
+
+          ./vic-tools/jenkins/jobs/vic-product-scenario/selenium_grid.sh create {version}
+
+          cat > build.envfile <<ENVS
+          PARALLEL_JOBS={parallel_jobs}
+          ROBOT_REPORT=report
+          NIMBUS_RETRY_ATTEMPTS=5
+          NIMBUS_RETRY_DELAY=1m
+          ESX_BUILD=$ESX_BUILD
+          VC_BUILD=$VC_BUILD
+          BUILD_TAG={repo}_${{BUILD_NUM}}
+          VIC_PRODUCT_BUILD=${{BUILD_NUM}}
+          VIC_PRODUCT_BRANCH={branch}
+          ENVS
+
+          docker run --net grid-{version} --privileged --rm --link selenium-hub-{version}:selenium-grid-hub -v /var/run/docker.sock:/var/run/docker.sock -v /etc/docker/certs.d:/etc/docker/certs.d -v "${{WORKSPACE}}":/go -v /vic-cache:/vic-cache --env-file "vic-internal/{repo}-scenario-{version}-secrets.list" --env-file build.envfile gcr.io/eminent-nation-87317/vic-integration-test:1.46 vic-tools/jenkins/jobs/vic-product-scenario/build.sh ${{TEST_CASES}}
+          ./vic-tools/jenkins/jobs/vic-product-scenario/selenium_grid.sh remove {version}

jenkins/jjb/templates/scenario-trigger.yaml

@@ -0,0 +1,15 @@
+- job-template:
+    name: '{repo}-{version}-trigger'
+    description: 'This is a parameterized build for running Nimbus scenario tests against build of {repo} {branch}'
+    node: 'vic-executor'
+    scm:
+      - git:
+          url: 'https://github.com/vmware/vic-tools'
+          branches:
+            - 'master'
+    triggers:
+      - timed: '@midnight'
+    builders:
+      - shell: |
+          #!/bin/bash
+          docker run  --rm -v ${{WORKSPACE}}:/src wdc-harbor-ci.eng.vmware.com/default-project/ci-trigger:0.0.1 /src/jenkins/jobs/scenario-trigger/build.sh {bucket} {branch} {prefix} {repo}-{version}-scenario

jenkins/jjb/views.yaml

@@ -0,0 +1,17 @@
+- view:
+    name: 'Scenario'
+    view-type: list
+    description: 'VIC Scenario Tests runs nightly'
+    job-name:
+      - vic-1.4.3-scenario
+      - vic-master-scenario
+      - vic-product-1.4.3-scenario
+      - vic-product-master-scenario
+    columns:
+      - status
+      - weather
+      - job
+      - last-success
+      - last-failure
+      - last-duration
+      - robot-list

jenkins/jobs/trigger/nightly_trigger.sh → jenkins/jobs/scenario-trigger/build.sh

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Usage example: ./nightly_trigger.sh vic-engine-builds master vic_ vic-master-nightly
+# Usage example: ./build.sh vic-engine-builds master vic_ vic-master-nightly
 
 set -ex
 
@@ -28,16 +28,26 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 # Get the latest build filename
 if [ "${REPO_BRANCH}" == "master" ]; then
-    GS_PATH="${ARTIFACT_BUCKET}/"
+    GS_PATH="${ARTIFACT_BUCKET}"
 else
-    GS_PATH="${ARTIFACT_BUCKET}/${REPO_BRANCH}/"
+    GS_PATH="${ARTIFACT_BUCKET}/${REPO_BRANCH}"
 fi
-FILE_NAME=$(gsutil ls -l "gs://${GS_PATH}${BINARY_PREFIX}*" | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | xargs basename)
+FILE_NAME=$(gsutil ls -l "gs://${GS_PATH}/${BINARY_PREFIX}*" | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | xargs basename)
 
 # strip prefix and suffix from archive filename
-BUILD_NUM=${FILE_NAME#${BINARY_PREFIX}}
-BUILD_NUM=${BUILD_NUM%%.*}
-
+case ${ARTIFACT_BUCKET} in
+    vic-engine-builds)
+        BUILD_NUM=${FILE_NAME#${BINARY_PREFIX}}
+        BUILD_NUM=${BUILD_NUM%%.*}
+        ;;
+    "vic-product-ova-builds")
+        BUILD_NUM=$(echo ${FILE_NAME} | awk -F '-' '{NF--;  print $NF }')
+        ;;
+    *)
+        echo "Bucket ${ARTIFACT_BUCKET} is not supported."
+        exit 1
+        ;;
+esac
 echo "Trigger build ${BUILD_NUM}"
 
 # Run test on vsphere 6.0, 6.5, 6.7 alternatively

jenkins/jobs/trigger/jenkins_job_trigger.py → jenkins/jobs/scenario-trigger/jenkins_job_trigger.py

@@ -5,12 +5,12 @@
 """
 
 import logging
-
 import argparse
+
 from jenkinsapi.custom_exceptions import NoBuildData
 from jenkinsapi.jenkins import Jenkins
 from jenkinsapi.api import log as jenkins_api_log
-
+from jenkinsapi.constants import STATUS_SUCCESS
 
 formatter = logging.Formatter('%(asctime)s | %(message)s', '%Y-%m-%d %H:%M:%S')
 console_handler = logging.StreamHandler()
@@ -24,9 +24,9 @@
 jenkins_api_log.addHandler(console_handler)
 
 
-def is_tested(jenkinsci, job_name, build_num):
+def is_verified(jenkinsci, job_name, build_num):
     """
-    Checks to see whether a specified build has already been tested.
+    Checks to see whether a specified build has already been verified.
     """
     try:
         job = jenkinsci[job_name]
@@ -43,6 +43,9 @@ def is_tested(jenkinsci, job_name, build_num):
         return False
 
     LOG.info('Build %s is already tested', build_num)
+    if build._data['result'] != STATUS_SUCCESS:
+        LOG.info('Build %s failed last time, rerun it', build_num)
+        return False
     return True
 
 
@@ -62,7 +65,7 @@ def main():
     args = parser.parse_args()
     jenkinsci = Jenkins(args.jenkins, username=args.username,
                         password=args.password)
-    if not is_tested(jenkinsci, args.job_name, args.build_num):
+    if not is_verified(jenkinsci, args.job_name, args.build_num):
         params = {'VSPHERE_VERSION': args.vsphere_version,
                   'ESX_BUILD': args.esx_build,
                   'VC_BUILD': args.vc_build,