Skip to content
This repository has been archived by the owner on Mar 1, 2022. It is now read-only.

Commit

Permalink
Add pr test jobs
Browse files Browse the repository at this point in the history
Add jenkins jobs for vic and vic-product PR testing. These
jobs are aimed to verify scenario test code.

Start selenium nodes according to parallel executor number
to avoid creating idle nodes.

Keep secret keywords for troubleshooting.
  • Loading branch information
DanielXiao committed Aug 31, 2018
1 parent 1c69838 commit 82949d6
Show file tree
Hide file tree
Showing 6 changed files with 202 additions and 26 deletions.
8 changes: 7 additions & 1 deletion jenkins/jjb/projects.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,10 @@
repo: 'vic-product'
version: 'master'
branch: 'master'
node: 'vic-executor1'
node: 'vic-executor1'
- 'vic-pr-test':
repo: 'vic'
node: 'vic-executor'
- 'vic-product-pr-test':
repo: 'vic-product'
node: 'vic-executor1'
182 changes: 175 additions & 7 deletions jenkins/jjb/templates/scenario-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,16 @@
branches:
- 'master'
basedir: 'vic-internal'
- git:
url: 'https://github.com/vmware/{repo}'
branches:
- '{branch}'
basedir: '{repo}'
- git:
url: 'https://github.com/vmware/vic-tools'
branches:
- 'master'
basedir: 'vic-tools'
- git:
url: 'https://github.com/vmware/{repo}'
branches:
- '{branch}'
basedir: '{repo}'
wrappers:
- timeout:
name: absolute_timeout
Expand Down Expand Up @@ -131,7 +131,7 @@
#!/bin/bash
set -x
./vic-tools/jenkins/jobs/vic-product-scenario/selenium_grid.sh create {version}
./vic-tools/jenkins/jobs/vic-product-scenario/selenium_grid.sh create {version} {parallel_jobs}
cat > build.envfile <<ENVS
PARALLEL_JOBS={parallel_jobs}
Expand All @@ -146,4 +146,172 @@
ENVS
docker run --net grid-{version} --privileged --rm --link selenium-hub-{version}:selenium-grid-hub -v /var/run/docker.sock:/var/run/docker.sock -v /etc/docker/certs.d:/etc/docker/certs.d -v "${{WORKSPACE}}":/go -v /vic-cache:/vic-cache --env-file "vic-internal/{repo}-scenario-{version}-secrets.list" --env-file build.envfile gcr.io/eminent-nation-87317/vic-integration-test:1.46 vic-tools/jenkins/jobs/vic-product-scenario/build.sh ${{TEST_CASES}}
./vic-tools/jenkins/jobs/vic-product-scenario/selenium_grid.sh remove {version}
./vic-tools/jenkins/jobs/vic-product-scenario/selenium_grid.sh remove {version} {parallel_jobs}
- defaults:
name: 'pr-tests'
description: 'This is a parameterized build for verifying your test code in a PR'
node: '{node}'
parameters:
- choice:
name: VSPHERE_VERSION
choices:
- '6.7'
- '6.5'
- '6.0'
description: 'The version of vSphere to run against. Skipped tests are determined by it.'
- string:
name: ESX_BUILD
default: '{default_esx_build}'
description: 'The build of ESX to deploy, e.g. ob-8169922 (6.7 default).'
- string:
name: VC_BUILD
default: '{default_vc_build}'
description: 'The build of VC to deploy, e.g. ob-8217866 (6.7 default).'
- string:
name: TEST_BRANCH
default: ''
description: 'Must specified. {repo} branch to check out tests from. May be a pull request, e.g. "pull/8063"'
- string:
name: BUILD_BRANCH
default: ''
description: 'This is {repo} branch to fetch build from google storage bucket, e.g. releases/1.4.3. master by default'
- string:
name: BUILD_NUM
default: ''
description: 'Build number to test. Get latest build in the google storage bucket folder if not specified'
- string:
name: NIMBUS_USER
default: ''
description: 'Must specified. The username of the nimbus user with which to run the tests'
- password:
name: NIMBUS_PASSWORD
default: ''
description: 'Must specified. The password of the nimbus user with which to run the tests'
- choice:
name: PARALLEL_JOBS
choices:
- '1'
- '2'
- '3'
- '4'
description: 'The parallel executor of robot tests.'
- string:
name: TEST_CASES
default: ''
description: |
The specific test cases to run as part of the job. These are passed as arguments to pabot so will likely be of the form "tests/manual-test-cases/Group23-Future-Tests"
Runs the default set of tests if left empty.
scm:
- git:
url: '[email protected]:core-build/vic-internal.git'
credentials-id: 'GitLabAutomationKey'
branches:
- 'master'
basedir: 'vic-internal'
- git:
url: 'https://github.com/vmware/vic-tools'
branches:
- 'master'
basedir: 'vic-tools'
- git:
url: 'https://github.com/vmware/{repo}'
branches:
- '${{TEST_BRANCH}}'
basedir: '{repo}'
name: ''
refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pull/*'
wrappers:
- timeout:
name: absolute_timeout
timeout: '{build_timeout}'
abort: true
type: absolute
- workspace-cleanup
publishers:
- robot:
output-path: '{repo}/report'
pass-threshold: '100'
unstable-threshold: '100'
- groovy-postbuild:
script: |
def vc_build = manager.envVars['VC_BUILD']
def esx_build = manager.envVars['ESX_BUILD']
def vsphere_version = manager.envVars.get('VSPHERE_VERSION')
def summary = manager.createSummary("gear2.gif")
summary.appendText("Infrastructure Builds:<ul>", false)
summary.appendText("<li><b>vSphere Version</b> - ${{vsphere_version}}</li>", false)
summary.appendText("<li><b>vCenter</b> - ${{vc_build}}</li>", false)
summary.appendText("<li><b>ESXi</b> - ${{esx_build}}</li>", false)
- email-ext:
success: true
aborted: true
failure: true
content-type: html
send-to:
- requester
subject: 'vmware/{repo} PR Testing - ${{TEST_BRANCH}}'
body: |
Download link for the test logs is in the <a href="${{BUILD_URL}}/console">build console output</a>
<p>
${{VSPHERE_VERSION}} Jenkins Nimbus Scenario Job: <a href="${{BUILD_URL}}">${{BUILD_URL}}</a>
<p>
Run Report: ${{ROBOT_REPORTLINK}}
<p>
${{FILE,path="{repo}/console.log"}}
- job-template:
name: 'vic-pr-test'
defaults: 'pr-tests'
builders:
- shell: |
#!/bin/bash
dest="vic-ci-logs/vmware/${{TEST_BRANCH}}/"
cat > build.envfile <<ENVS
PARALLEL_JOBS=${{PARALLEL_JOBS}}
NIMBUS_RETRY_ATTEMPTS=5
NIMBUS_RETRY_DELAY=1m
LOG_UPLOAD_DEST=${{dest}}
GIT_COMMIT=${{GIT_COMMIT}}
BUILD_TIMESTAMP=${{BUILD_TIMESTAMP}}
BUILD_ID=${{BUILD_ID}}
ESX_BUILD=${{ESX_BUILD}}
VC_BUILD=${{VC_BUILD}}
BUILD_TAG=vic_${{BUILD_NUM}}
VCH_BUILD=${{BUILD_NUM}}
VCH_BRANCH=${{BUILD_BRANCH}}
ENVS
docker run --rm -v "${{WORKSPACE}}":/go --env-file vic-internal/{repo}-scenario-master-secrets.list --env-file build.envfile -e NIMBUS_USER="${{NIMBUS_USER}}" -e NIMBUS_PASSWORD="${{NIMBUS_PASSWORD}}" --name=${{BUILD_TAG}} gcr.io/eminent-nation-87317/vic-integration-test:1.48 vic-tools/jenkins/jobs/vic-scenario/build.sh ${{VSPHERE_VERSION}} ${{TEST_CASES}}
- job-template:
name: 'vic-product-pr-test'
defaults: 'pr-tests'
builders:
- shell: |
#!/bin/bash
set -x
if [ -z "${{TEST_BRANCH}}" ] || [ -z "${{NIMBUS_USER}}" ] || [ -z "${{NIMBUS_PASSWORD}}" ]; then
echo "TEST_BRANCH, NIMBUS_USER and NIMBUS_PASSWORD are must specified"
exit 1
fi
./vic-tools/jenkins/jobs/vic-product-scenario/selenium_grid.sh create ${{BUILD_ID}} ${{PARALLEL_JOBS}}
cat > build.envfile <<ENVS
PARALLEL_JOBS=${{PARALLEL_JOBS}}
ROBOT_REPORT=report
NIMBUS_RETRY_ATTEMPTS=5
NIMBUS_RETRY_DELAY=1m
ESX_BUILD=${{ESX_BUILD}}
VC_BUILD=${{VC_BUILD}}
BUILD_TAG=vic-product_${{BUILD_NUM}}
VIC_PRODUCT_BUILD=${{BUILD_NUM}}
VIC_PRODUCT_BRANCH=${{BUILD_BRANCH}}
ENVS
set +x
docker run --net grid-${{BUILD_ID}} --privileged --rm --link selenium-hub-${{BUILD_ID}}:selenium-grid-hub -v /var/run/docker.sock:/var/run/docker.sock -v /etc/docker/certs.d:/etc/docker/certs.d -v "${{WORKSPACE}}":/go -v /vic-cache:/vic-cache --env-file "vic-internal/{repo}-scenario-master-secrets.list" --env-file build.envfile -e NIMBUS_USER="${{NIMBUS_USER}}" -e NIMBUS_PASSWORD="${{NIMBUS_PASSWORD}}" gcr.io/eminent-nation-87317/vic-integration-test:1.46 vic-tools/jenkins/jobs/vic-product-scenario/build.sh ${{TEST_CASES}}
set -x
./vic-tools/jenkins/jobs/vic-product-scenario/selenium_grid.sh remove ${{BUILD_ID}} ${{PARALLEL_JOBS}}
3 changes: 3 additions & 0 deletions jenkins/jjb/views.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
- vic-master-scenario
- vic-product-1.4.3-scenario
- vic-product-master-scenario
- vic-product-pr-test
- vic-pr-test
columns:
- status
- weather
Expand All @@ -15,3 +17,4 @@
- last-failure
- last-duration
- robot-list
- build-button
6 changes: 3 additions & 3 deletions jenkins/jobs/vic-product-scenario/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ VC_67_VERSION="ob-8217866"

DEFAULT_TESTCASES=("tests/manual-test-cases")

DEFAULT_VIC_PRODUCT_BRANCH=""
DEFAULT_VIC_PRODUCT_BRANCH="master"
DEFAULT_VIC_PRODUCT_BUILD="*"

DEFAULT_PARALLEL_JOBS=4
Expand Down Expand Up @@ -61,7 +61,7 @@ ${GCS_BUCKET:="vic-product-ova-builds"}

VIC_PRODUCT_BRANCH=${VIC_PRODUCT_BRANCH:-${DEFAULT_VIC_PRODUCT_BRANCH}}
VIC_PRODUCT_BUILD=${VIC_PRODUCT_BUILD:-${DEFAULT_VIC_PRODUCT_BUILD}}
if [ "${VIC_PRODUCT_BRANCH}" == "master" ]; then
if [ "${VIC_PRODUCT_BRANCH}" == "${DEFAULT_VIC_PRODUCT_BRANCH}" ]; then
GS_PATH="${GCS_BUCKET}"
else
GS_PATH="${GCS_BUCKET}/${VIC_PRODUCT_BRANCH}"
Expand Down Expand Up @@ -91,7 +91,7 @@ pushd ${WORKSPACE_DIR}/vic-product
echo "VIC Product OVA download complete..."

PARALLEL_JOBS=${PARALLEL_JOBS:-${DEFAULT_PARALLEL_JOBS}}
pabot --verbose --processes "${PARALLEL_JOBS}" -d report --removekeywords TAG:secret "${excludes[@]}" --variable ESX_VERSION:"${ESX_BUILD}" --variable VC_VERSION:"${VC_BUILD}" "${testcases[@]}"
pabot --verbose --processes "${PARALLEL_JOBS}" -d report "${excludes[@]}" --variable ESX_VERSION:"${ESX_BUILD}" --variable VC_VERSION:"${VC_BUILD}" "${testcases[@]}"
cat report/pabot_results/*/stdout.txt | grep -E '::|\.\.\.' | grep -E 'PASS|FAIL' > console.log

# Pretty up the email results
Expand Down
23 changes: 11 additions & 12 deletions jenkins/jobs/vic-product-scenario/selenium_grid.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,23 @@ start_node () {

remove_all () {
echo "Kill any old selenium infrastructure..."
docker rm -f ${HUB_NM} ${NODE1_NM} ${NODE2_NM} ${NODE3_NM} ${NODE4_NM}
docker rm -f ${HUB_NM}
for i in $(seq 1 ${COUNT_NUM}); do
docker rm -f "firefox${i}-${SUFFIX}"
done
docker network prune -f
}

if [ $# -ne 2 ] ; then
echo "Usage: $0 create/remove suffix"
if [ $# -ne 3 ] ; then
echo "Usage: $0 create/remove suffix count"
exit 1
fi

ACTION=$1
SUFFIX=$2
COUNT_NUM=$3
NETWORK="grid-${SUFFIX}"
HUB_NM="selenium-hub-${SUFFIX}"
NODE1_NM="firefox1-${SUFFIX}"
NODE2_NM="firefox2-${SUFFIX}"
NODE3_NM="firefox3-${SUFFIX}"
NODE4_NM="firefox4-${SUFFIX}"

case ${ACTION} in
create)
Expand All @@ -59,11 +59,10 @@ case ${ACTION} in
fi
sleep 3;
done

start_node ${NODE1_NM} selenium/node-firefox:3.9.1
start_node ${NODE2_NM} selenium/node-firefox:3.9.1
start_node ${NODE3_NM} selenium/node-firefox:3.9.1
start_node ${NODE4_NM} selenium/node-firefox:3.9.1

for i in $(seq 1 ${COUNT_NUM}); do
start_node "firefox${i}-${SUFFIX}" selenium/node-firefox:3.9.1
done
;;
remove)
remove_all
Expand Down
6 changes: 3 additions & 3 deletions jenkins/jobs/vic-scenario/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ VC_67_VERSION="ob-8217866"

DEFAULT_LOG_UPLOAD_DEST="vic-ci-logs"
DEFAULT_ARTIFACT_BUCKET="vic-engine-builds"
DEFAULT_VCH_BRANCH=""
DEFAULT_VCH_BRANCH="master"
DEFAULT_VCH_BUILD="*"
DEFAULT_TESTCASES=("tests/manual-test-cases/Group5-Functional-Tests" "tests/manual-test-cases/Group13-vMotion" "tests/manual-test-cases/Group21-Registries" "tests/manual-test-cases/Group23-Future-Tests")

Expand Down Expand Up @@ -65,7 +65,7 @@ testcases=("${@:-${DEFAULT_TESTCASES[@]}}")
VCH_BUILD=${VCH_BUILD:-${DEFAULT_VCH_BUILD}}
VCH_BRANCH=${VCH_BRANCH:-${DEFAULT_VCH_BRANCH}}
ARTIFACT_BUCKET=${ARTIFACT_BUCKET:-${DEFAULT_ARTIFACT_BUCKET}}
if [ "${VCH_BRANCH}" == "master" ]; then
if [ "${VCH_BRANCH}" == "${DEFAULT_VCH_BRANCH}" ]; then
GS_PATH="${ARTIFACT_BUCKET}"
else
GS_PATH="${ARTIFACT_BUCKET}/${VCH_BRANCH}"
Expand Down Expand Up @@ -125,7 +125,7 @@ pushd vic
echo "Tarball extraction passed, Running nightlies test.."
fi

pabot --processes ${PARALLEL_JOBS} --removekeywords TAG:secret ${excludes} --variable ESX_VERSION:"${ESX_BUILD}" --variable VC_VERSION:"${VC_BUILD}" -d report "${testcases[@]}"
pabot --processes ${PARALLEL_JOBS} ${excludes} --variable ESX_VERSION:"${ESX_BUILD}" --variable VC_VERSION:"${VC_BUILD}" -d report "${testcases[@]}"
cat report/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log

# See if any VMs leaked
Expand Down

0 comments on commit 82949d6

Please sign in to comment.