Skip to content

Commit

Permalink
Test ITN for messagin service
Browse files Browse the repository at this point in the history
  • Loading branch information
@sahusanket
sahusanket committed Dec 12, 2024
1 parent e631442 commit 7e469b9
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 83 deletions.
128 changes: 46 additions & 82 deletions .github/workflows/build-and-unit-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,36 +11,14 @@

name: Build and Unit Test
on:
schedule:
- cron: '0 8 * * *'
workflow_dispatch:
# workaround to run manual trigger for a particular branch
inputs:
branch:
description: "Branch name on which workflow will be triggered"
required: true
default: "develop"
skip_tests:
description: "Skip running tests for this build"
required: true
type: boolean
default: false
run_tests_with_retries:
description: "Run tests with retries, it will be ignored if tests are skipped."
required: true
type: boolean
default: false
keep_existing_release:
description: "Do not overwrite the GitHub release for versions without snapshot"
required: true
type: boolean
default: true
push

env:
TAG_NAME: latest
PRE_RELEASE: true
REPLACE_ARTIFACTS: true
REMOVE_ARTIFACTS: true
SKIP_RELEASE: false
MAX_ATTEMPTS: 3

jobs:
Expand All @@ -53,7 +31,7 @@ jobs:
run: |
if [ ${{ github.event_name }} != "workflow_dispatch" ];
then
echo 'BRANCH_MATRIX={"include":[{"branch":"develop"}, {"branch":"release/6.7"}, {"branch":"release/6.8"}, {"branch":"release/6.9"}]}' >> $GITHUB_ENV
echo 'BRANCH_MATRIX={"include":[{"branch":"int_610_fix"}]}' >> $GITHUB_ENV
else
echo 'BRANCH_MATRIX={"include":[{"branch":"${{ github.event.inputs.branch }}"}]}' >> $GITHUB_ENV
if [ "${{ github.event.inputs.run_tests_with_retries }}" == "false" ]
Expand All @@ -80,7 +58,7 @@ jobs:

- name: Get Secrets from GCP Secret Manager
id: 'secrets'
uses: 'google-github-actions/get-secretmanager-secrets@v0'
uses: 'google-github-actions/get-secretmanager-secrets@v2'
if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
with:
secrets: |-
Expand All @@ -90,7 +68,7 @@ jobs:
CDAP_GPG_PRIVATE_KEY:cdapio-github-builds/CDAP_GPG_PRIVATE_KEY
- name: Recursively Checkout Repository
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive
Expand All @@ -103,7 +81,7 @@ jobs:
git submodule update --init --recursive --remote
- name: Cache
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
Expand All @@ -116,40 +94,6 @@ jobs:
name=$(echo -n "${{ matrix.branch }}" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g')
echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV
- name: Run Tests
if: "${{ github.event.inputs.skip_tests != 'true' }}"
# Pinned version 2.8.2
uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482
with:
timeout_minutes: 120
max_attempts: ${{ needs.set-branch-matrix.outputs.retries }}
retry_on: error
on_retry_command: echo "Tests failed in this attempt, retrying ..."
command: |
cd cdap-build
MAVEN_OPTS="-Xmx16G -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/cdap-build/oom.bin" mvn test -fae -T2 -U -V -am -amd -P templates,unit-tests -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
- name: Archive build artifacts
uses: actions/upload-artifact@v3
if: "${{ github.event.inputs.skip_tests != 'true' || failure() }}"
with:
name: Build debug files - ${{ env.ARTIFACT_NAME }}
path: |
**/target/rat.txt
**/target/surefire-reports/*
/cdap-build/oom.bin
- name: Surefire Report
# Pinned 3.5.2 version
uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32
if: ${{ always() }}
with:
report_paths: '**/target/surefire-reports/TEST-*.xml'
github_token: ${{ secrets.GITHUB_TOKEN }}
detailed_summary: true
commit: ${{ github.sha }}
check_name: Test Report - ${{ env.ARTIFACT_NAME }}

- name: Build Standalone
# Pinned version 2.8.2
uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482
Expand All @@ -159,8 +103,8 @@ jobs:
retry_on: error
on_retry_command: echo "Build Standalone failed in this attempt, retrying ..."
command: |
cd cdap-build
MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
cd cdap-build
MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
- name: Find Build Version
working-directory: cdap-build/cdap
Expand All @@ -173,14 +117,15 @@ jobs:
echo "PRE_RELEASE=false" >> $GITHUB_ENV
echo "REPLACE_ARTIFACTS=false" >> $GITHUB_ENV
echo "REMOVE_ARTIFACTS=false" >> $GITHUB_ENV
echo "SKIP_RELEASE=true" >> $GITHUB_ENV
echo "Release will not be overwritten if exists."
else
echo "Release will be overwritten if exists."
fi
- name: Upload CDAP Standalone
if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
uses: actions/upload-artifact@v3 # https://github.com/actions/upload-artifact#zipped-artifact-downloads
uses: actions/upload-artifact@v4 # https://github.com/actions/upload-artifact#zipped-artifact-downloads
with:
name: cdap-sandbox-${{env.CDAP_VERSION}}.zip
path: cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip
Expand All @@ -200,21 +145,39 @@ jobs:
GPG_PRIVATE_KEY: ${{ steps.secrets.outputs.CDAP_GPG_PRIVATE_KEY }}

- name: Maven Deploy
# Pinned version 2.8.2
uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482
with:
timeout_minutes: 120
max_attempts: 3
retry_on: error
on_retry_command: echo "Maven Deploy failed in this attempt, retrying ..."
command: |
cd cdap-build
if [[ (${{ matrix.branch }} == "develop") || (${{ matrix.branch }} == release/*) ]];
run: |
cd cdap-build
if [[ (${{ matrix.branch }} == "develop") || (${{ matrix.branch }} == release/*) ]];
then
retry_count=0
failed_module=""
while [ $retry_count -lt 3 ]; do
if [ -n "$failed_module" ];
then
build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE -rf :"$failed_module" 2>&1 || true)
else
build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE 2>&1 || true)
fi
echo "$build_output"
if [ $(echo "$build_output" | grep -c "BUILD FAILURE") -gt 0 ];
then
echo "[WARNING] Deployment failed, retrying..."
failed_module=$(echo "$build_output" | grep -e 'mvn <args> -rf' | sed -n 's/.*mvn <args> -rf ://p')
echo "[INFO] FAILED MODULE = $failed_module"
retry_count=$((retry_count + 1))
else
echo "[INFO] Deployment successful"
break
fi
done
if [ $retry_count -ge 3 ];
then
mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE
else
mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
echo "[ERROR] Max retries reached..., deployment failed"
exit 1
fi
else
mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
fi
env:
CDAP_OSSRH_USERNAME: ${{ steps.secrets.outputs.CDAP_OSSRH_USERNAME }}
CDAP_OSSRH_PASSWORD: ${{ steps.secrets.outputs.CDAP_OSSRH_PASSWORD }}
Expand Down Expand Up @@ -251,19 +214,20 @@ jobs:
fi
- name: Upload CDAP Standalone and CDAP DEB Bundle
# Pinned 1.11.1 version
uses: ncipollo/release-action@4c75f0f2e4ae5f3c807cf0904605408e319dcaac
# Pinned 1.14.0 version
uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5
with:
allowUpdates: true
omitBodyDuringUpdate: true
omitNameDuringUpdate: true
omitPrereleaseDuringUpdate: true
skipIfReleaseExists: ${{ env.SKIP_RELEASE }}
prerelease: ${{ env.PRE_RELEASE }}
removeArtifacts: ${{ env.REMOVE_ARTIFACTS }}
replacesArtifacts: ${{ env.REPLACE_ARTIFACTS }}
token: ${{ secrets.GITHUB_TOKEN }}
tag: ${{ env.TAG_NAME }}
body: Cask Data Appplication Platform - Release ${{ env.CDAP_VERSION }}
body: Cask Data Application Platform - Release ${{ env.CDAP_VERSION }}
artifacts: |
cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip,cdap-build/cdap/cdap-distributions/target/cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz
Expand All @@ -279,4 +243,4 @@ jobs:
- name: Report Result
if: ${{ (matrix.branch == 'develop' || startsWith(matrix.branch, 'release/')) && github.event.inputs.skip_tests != 'true' && always() }}
run: |
echo '{ "repository": "${{ github.repository }}", "build_name": "${{ github.workflow }}", "branch": "${{ matrix.branch }}", "conclusion": "${{ job.status }}", "started_at": "'$STARTED_AT'", "ended_at": "'$ENDED_AT'", "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", "run_id": "${{ github.run_id }}", "run_attempt": "${{ github.run_attempt }}" }' | bq insert ${{ vars.BUILD_HISTORY_TABLE }}
echo '{ "repository": "${{ github.repository }}", "build_name": "${{ github.workflow }}", "branch": "${{ matrix.branch }}", "conclusion": "${{ job.status }}", "started_at": "'$STARTED_AT'", "ended_at": "'$ENDED_AT'", "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", "run_id": "${{ github.run_id }}", "run_attempt": "${{ github.run_attempt }}" }' | bq insert ${{ vars.BUILD_HISTORY_TABLE }}
2 changes: 1 addition & 1 deletion .gitmodules
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[submodule "cdap"]
path = cdap
url = ../cdap.git
branch = release/6.10
branch = itn_messaging_fix
[submodule "app-artifacts/hydrator-plugins"]
path = app-artifacts/hydrator-plugins
url = ../hydrator-plugins.git
Expand Down

0 comments on commit 7e469b9

Please sign in to comment.