Skip to content

Build and Unit Test #1559

Build and Unit Test

Build and Unit Test #1559

# Copyright © 2022 Cask Data, Inc.
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
name: Build and Unit Test
on:
schedule:
- cron: '0 8 * * *'
workflow_dispatch:
# workaround to run manual trigger for a particular branch
inputs:
branch:
description: "Branch name on which workflow will be triggered"
required: true
default: "develop"
skip_tests:
description: "Skip running tests for this build"
required: true
type: boolean
default: false
run_tests_with_retries:
description: "Run tests with retries, it will be ignored if tests are skipped."
required: true
type: boolean
default: false
keep_existing_release:
description: "Do not overwrite the GitHub release for versions without snapshot"
required: true
type: boolean
default: true
env:
TAG_NAME: latest
PRE_RELEASE: true
REPLACE_ARTIFACTS: true
REMOVE_ARTIFACTS: true
SKIP_RELEASE: false
MAX_ATTEMPTS: 3
jobs:
set-branch-matrix:
runs-on: ubuntu-latest
steps:
- name: set-matrix
run: |
if [ ${{ github.event_name }} != "workflow_dispatch" ];
then
echo 'BRANCH_MATRIX={"include":[{"branch":"develop"}, {"branch":"release/6.9"}, {"branch":"release/6.10"}]}' >> $GITHUB_ENV
else
echo 'BRANCH_MATRIX={"include":[{"branch":"${{ github.event.inputs.branch }}"}]}' >> $GITHUB_ENV
if [ "${{ github.event.inputs.run_tests_with_retries }}" == "false" ]
then
echo 'MAX_ATTEMPTS=1' >> $GITHUB_ENV
fi
fi
outputs:
matrix: ${{ env.BRANCH_MATRIX }}
retries: ${{ env. MAX_ATTEMPTS }}
build:
needs: set-branch-matrix
runs-on: cdapio-hub-k8-runner
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.set-branch-matrix.outputs.matrix) }}
steps:
- name: Get Start Time
run: echo "STARTED_AT=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV
- name: Get Secrets from GCP Secret Manager
id: 'secrets'
uses: 'google-github-actions/get-secretmanager-secrets@v2'
if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
with:
secrets: |-
CDAP_OSSRH_USERNAME:cdapio-github-builds/CDAP_OSSRH_USERNAME
CDAP_OSSRH_PASSWORD:cdapio-github-builds/CDAP_OSSRH_PASSWORD
CDAP_GPG_PASSPHRASE:cdapio-github-builds/CDAP_GPG_PASSPHRASE
CDAP_GPG_PRIVATE_KEY:cdapio-github-builds/CDAP_GPG_PRIVATE_KEY
- name: Recursively Checkout Repository
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive
path: cdap-build
ref: ${{ matrix.branch }}
- name: Update Submodules
working-directory: cdap-build
run: |
git submodule update --init --recursive --remote
- name: Cache
uses: actions/cache@v4
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-${{ github.workflow }}
- name: Set up Artifact Name # Removes invalid artifact name characters: ",:,<,>,|,*,?,\,/.
if: "${{ github.event.inputs.skip_tests != 'true' }}"
run: |
name=$(echo -n "${{ matrix.branch }}" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g')
echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV
- name: Run Tests
if: "${{ github.event.inputs.skip_tests != 'true' }}"
# Pinned version 2.8.2
uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482
with:
timeout_minutes: 120
max_attempts: ${{ needs.set-branch-matrix.outputs.retries }}
retry_on: error
on_retry_command: echo "Tests failed in this attempt, retrying ..."
command: |
cd cdap-build
MAVEN_OPTS="-Xmx16G -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/cdap-build/oom.bin" mvn test -fae -T2 -U -V -am -amd -P templates,unit-tests -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
- name: Archive build artifacts
uses: actions/upload-artifact@v4
if: "${{ github.event.inputs.skip_tests != 'true' || failure() }}"
with:
name: Build debug files - ${{ env.ARTIFACT_NAME }}
path: |
**/target/rat.txt
**/target/surefire-reports/*
/cdap-build/oom.bin
- name: Surefire Report
# Pinned 3.5.2 version
uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32
if: ${{ always() }}
with:
report_paths: '**/target/surefire-reports/TEST-*.xml'
github_token: ${{ secrets.GITHUB_TOKEN }}
detailed_summary: true
commit: ${{ github.sha }}
check_name: Test Report - ${{ env.ARTIFACT_NAME }}
- name: Build Standalone
# Pinned version 2.8.2
uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482
with:
timeout_minutes: 60
max_attempts: 3
retry_on: error
on_retry_command: echo "Build Standalone failed in this attempt, retrying ..."
command: |
cd cdap-build
MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
- name: Find Build Version
working-directory: cdap-build/cdap
run: |
export VERSION=$(ls cdap-standalone/target/cdap-sandbox*zip | cut --delimiter=- --fields="-3" --complement | rev | cut --delimiter="." --fields=1 --complement | rev)
echo "CDAP_VERSION=${VERSION}"
echo "CDAP_VERSION=${VERSION}" >> $GITHUB_ENV
if [[ ($VERSION != *-SNAPSHOT) && ("${{ github.event.inputs.keep_existing_release }}" != "false") ]];
then
echo "PRE_RELEASE=false" >> $GITHUB_ENV
echo "REPLACE_ARTIFACTS=false" >> $GITHUB_ENV
echo "REMOVE_ARTIFACTS=false" >> $GITHUB_ENV
echo "SKIP_RELEASE=true" >> $GITHUB_ENV
echo "Release will not be overwritten if exists."
else
echo "Release will be overwritten if exists."
fi
- name: Upload CDAP Standalone
if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
uses: actions/upload-artifact@v4 # https://github.com/actions/upload-artifact#zipped-artifact-downloads
with:
name: cdap-sandbox-${{env.CDAP_VERSION}}.zip
path: cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip
- name: Set up GPG conf
if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
run: |
echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf
- name: Import GPG key
if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
run: |
echo "$GPG_PRIVATE_KEY" > private.key
gpg --import --batch private.key
env:
GPG_PRIVATE_KEY: ${{ steps.secrets.outputs.CDAP_GPG_PRIVATE_KEY }}
- name: Maven Deploy
run: |
cd cdap-build
if [[ (${{ matrix.branch }} == "develop") || (${{ matrix.branch }} == release/*) ]];
then
retry_count=0
failed_module=""
while [ $retry_count -lt 3 ]; do
if [ -n "$failed_module" ];
then
build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE -rf :"$failed_module" 2>&1 || true)
else
build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE 2>&1 || true)
fi
echo "$build_output"
if [ $(echo "$build_output" | grep -c "BUILD FAILURE") -gt 0 ];
then
echo "[WARNING] Deployment failed, retrying..."
failed_module=$(echo "$build_output" | grep -e 'mvn <args> -rf' | sed -n 's/.*mvn <args> -rf ://p')
echo "[INFO] FAILED MODULE = $failed_module"
retry_count=$((retry_count + 1))
else
echo "[INFO] Deployment successful"
break
fi
done
if [ $retry_count -ge 3 ];
then
echo "[ERROR] Max retries reached..., deployment failed"
exit 1
fi
else
mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
fi
env:
CDAP_OSSRH_USERNAME: ${{ steps.secrets.outputs.CDAP_OSSRH_USERNAME }}
CDAP_OSSRH_PASSWORD: ${{ steps.secrets.outputs.CDAP_OSSRH_PASSWORD }}
CDAP_GPG_PASSPHRASE: ${{ steps.secrets.outputs.CDAP_GPG_PASSPHRASE }}
MAVEN_OPTS: "-Xmx12G"
- name: Build DEB Bundle
working-directory: cdap-build/cdap
run: |
mkdir -p cdap-distributions/target/deb-bundle-tmp
cd cdap-distributions/target/deb-bundle-tmp
cp ../../../*/target/*.deb .
tar zcf ../cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz *.deb
- name: Set Up Tag
working-directory: cdap-build
run: |
if [[ ${{ matrix.branch }} == release/* ]];
then
echo "TAG_NAME=v$CDAP_VERSION" >> $GITHUB_ENV
echo "TAG_NAME=v$CDAP_VERSION"
git tag -f v$CDAP_VERSION
git push -f origin refs/tags/v$CDAP_VERSION:refs/tags/v$CDAP_VERSION
elif [[ ${{ matrix.branch }} != "develop" ]];
then
export TAG=$(git check-ref-format --normalize "tags/${{ matrix.branch }}/tag" | cut -c6- | rev | cut -c5- | rev)
echo "TAG_NAME=$TAG" >> $GITHUB_ENV
echo "TAG_NAME=$TAG"
git tag -f $TAG
git push -f origin refs/tags/$TAG:refs/tags/$TAG
else
git tag -f ${{ env.TAG_NAME }}
git push -f origin refs/tags/${{ env.TAG_NAME }}:refs/tags/${{ env.TAG_NAME }}
fi
- name: Upload CDAP Standalone and CDAP DEB Bundle
# Pinned 1.14.0 version
uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5
with:
allowUpdates: true
omitBodyDuringUpdate: true
omitNameDuringUpdate: true
omitPrereleaseDuringUpdate: true
skipIfReleaseExists: ${{ env.SKIP_RELEASE }}
prerelease: ${{ env.PRE_RELEASE }}
removeArtifacts: ${{ env.REMOVE_ARTIFACTS }}
replacesArtifacts: ${{ env.REPLACE_ARTIFACTS }}
token: ${{ secrets.GITHUB_TOKEN }}
tag: ${{ env.TAG_NAME }}
body: Cask Data Application Platform - Release ${{ env.CDAP_VERSION }}
artifacts: |
cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip,cdap-build/cdap/cdap-distributions/target/cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz
- name: Alert team if build fails
if: ${{ (matrix.branch == 'develop' || startsWith(matrix.branch, 'release/')) && failure() }}
run: |
gcloud logging write cdapio-github-builds-logs '{ "message": "Build failure", "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' --payload-type=json --severity=ERROR
- name: Get End Time
if: ${{ always() }}
run: echo "ENDED_AT=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV
- name: Report Result
if: ${{ (matrix.branch == 'develop' || startsWith(matrix.branch, 'release/')) && github.event.inputs.skip_tests != 'true' && always() }}
run: |
echo '{ "repository": "${{ github.repository }}", "build_name": "${{ github.workflow }}", "branch": "${{ matrix.branch }}", "conclusion": "${{ job.status }}", "started_at": "'$STARTED_AT'", "ended_at": "'$ENDED_AT'", "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", "run_id": "${{ github.run_id }}", "run_attempt": "${{ github.run_attempt }}" }' | bq insert ${{ vars.BUILD_HISTORY_TABLE }}