From 7e469b95d8f91c0cf9532ccf82b137888210668e Mon Sep 17 00:00:00 2001 From: sahusanket Date: Thu, 12 Dec 2024 15:03:58 +0530 Subject: [PATCH] Test ITN for messagin service --- .github/workflows/build-and-unit-test.yml | 128 ++++++++-------------- .gitmodules | 2 +- 2 files changed, 47 insertions(+), 83 deletions(-) diff --git a/.github/workflows/build-and-unit-test.yml b/.github/workflows/build-and-unit-test.yml index b7b7d901..01d3bb55 100644 --- a/.github/workflows/build-and-unit-test.yml +++ b/.github/workflows/build-and-unit-test.yml @@ -11,36 +11,14 @@ name: Build and Unit Test on: - schedule: - - cron: '0 8 * * *' - workflow_dispatch: - # workaround to run manual trigger for a particular branch - inputs: - branch: - description: "Branch name on which workflow will be triggered" - required: true - default: "develop" - skip_tests: - description: "Skip running tests for this build" - required: true - type: boolean - default: false - run_tests_with_retries: - description: "Run tests with retries, it will be ignored if tests are skipped." - required: true - type: boolean - default: false - keep_existing_release: - description: "Do not overwrite the GitHub release for versions without snapshot" - required: true - type: boolean - default: true + push env: TAG_NAME: latest PRE_RELEASE: true REPLACE_ARTIFACTS: true REMOVE_ARTIFACTS: true + SKIP_RELEASE: false MAX_ATTEMPTS: 3 jobs: @@ -53,7 +31,7 @@ jobs: run: | if [ ${{ github.event_name }} != "workflow_dispatch" ]; then - echo 'BRANCH_MATRIX={"include":[{"branch":"develop"}, {"branch":"release/6.7"}, {"branch":"release/6.8"}, {"branch":"release/6.9"}]}' >> $GITHUB_ENV + echo 'BRANCH_MATRIX={"include":[{"branch":"int_610_fix"}]}' >> $GITHUB_ENV else echo 'BRANCH_MATRIX={"include":[{"branch":"${{ github.event.inputs.branch }}"}]}' >> $GITHUB_ENV if [ "${{ github.event.inputs.run_tests_with_retries }}" == "false" ] @@ -80,7 +58,7 @@ jobs: - name: Get Secrets from GCP Secret Manager id: 'secrets' - uses: 'google-github-actions/get-secretmanager-secrets@v0' + uses: 'google-github-actions/get-secretmanager-secrets@v2' if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} with: secrets: |- @@ -90,7 +68,7 @@ jobs: CDAP_GPG_PRIVATE_KEY:cdapio-github-builds/CDAP_GPG_PRIVATE_KEY - name: Recursively Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 submodules: recursive @@ -103,7 +81,7 @@ jobs: git submodule update --init --recursive --remote - name: Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }} @@ -116,40 +94,6 @@ jobs: name=$(echo -n "${{ matrix.branch }}" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g') echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV - - name: Run Tests - if: "${{ github.event.inputs.skip_tests != 'true' }}" - # Pinned version 2.8.2 - uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482 - with: - timeout_minutes: 120 - max_attempts: ${{ needs.set-branch-matrix.outputs.retries }} - retry_on: error - on_retry_command: echo "Tests failed in this attempt, retrying ..." - command: | - cd cdap-build - MAVEN_OPTS="-Xmx16G -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/cdap-build/oom.bin" mvn test -fae -T2 -U -V -am -amd -P templates,unit-tests -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 - - - name: Archive build artifacts - uses: actions/upload-artifact@v3 - if: "${{ github.event.inputs.skip_tests != 'true' || failure() }}" - with: - name: Build debug files - ${{ env.ARTIFACT_NAME }} - path: | - **/target/rat.txt - **/target/surefire-reports/* - /cdap-build/oom.bin - - - name: Surefire Report - # Pinned 3.5.2 version - uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32 - if: ${{ always() }} - with: - report_paths: '**/target/surefire-reports/TEST-*.xml' - github_token: ${{ secrets.GITHUB_TOKEN }} - detailed_summary: true - commit: ${{ github.sha }} - check_name: Test Report - ${{ env.ARTIFACT_NAME }} - - name: Build Standalone # Pinned version 2.8.2 uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482 @@ -159,8 +103,8 @@ jobs: retry_on: error on_retry_command: echo "Build Standalone failed in this attempt, retrying ..." command: | - cd cdap-build - MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 + cd cdap-build + MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 - name: Find Build Version working-directory: cdap-build/cdap @@ -173,6 +117,7 @@ jobs: echo "PRE_RELEASE=false" >> $GITHUB_ENV echo "REPLACE_ARTIFACTS=false" >> $GITHUB_ENV echo "REMOVE_ARTIFACTS=false" >> $GITHUB_ENV + echo "SKIP_RELEASE=true" >> $GITHUB_ENV echo "Release will not be overwritten if exists." else echo "Release will be overwritten if exists." @@ -180,7 +125,7 @@ jobs: - name: Upload CDAP Standalone if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} - uses: actions/upload-artifact@v3 # https://github.com/actions/upload-artifact#zipped-artifact-downloads + uses: actions/upload-artifact@v4 # https://github.com/actions/upload-artifact#zipped-artifact-downloads with: name: cdap-sandbox-${{env.CDAP_VERSION}}.zip path: cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip @@ -200,21 +145,39 @@ jobs: GPG_PRIVATE_KEY: ${{ steps.secrets.outputs.CDAP_GPG_PRIVATE_KEY }} - name: Maven Deploy - # Pinned version 2.8.2 - uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482 - with: - timeout_minutes: 120 - max_attempts: 3 - retry_on: error - on_retry_command: echo "Maven Deploy failed in this attempt, retrying ..." - command: | - cd cdap-build - if [[ (${{ matrix.branch }} == "develop") || (${{ matrix.branch }} == release/*) ]]; + run: | + cd cdap-build + if [[ (${{ matrix.branch }} == "develop") || (${{ matrix.branch }} == release/*) ]]; + then + retry_count=0 + failed_module="" + while [ $retry_count -lt 3 ]; do + if [ -n "$failed_module" ]; + then + build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE -rf :"$failed_module" 2>&1 || true) + else + build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE 2>&1 || true) + fi + echo "$build_output" + if [ $(echo "$build_output" | grep -c "BUILD FAILURE") -gt 0 ]; + then + echo "[WARNING] Deployment failed, retrying..." + failed_module=$(echo "$build_output" | grep -e 'mvn -rf' | sed -n 's/.*mvn -rf ://p') + echo "[INFO] FAILED MODULE = $failed_module" + retry_count=$((retry_count + 1)) + else + echo "[INFO] Deployment successful" + break + fi + done + if [ $retry_count -ge 3 ]; then - mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE - else - mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true + echo "[ERROR] Max retries reached..., deployment failed" + exit 1 fi + else + mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true + fi env: CDAP_OSSRH_USERNAME: ${{ steps.secrets.outputs.CDAP_OSSRH_USERNAME }} CDAP_OSSRH_PASSWORD: ${{ steps.secrets.outputs.CDAP_OSSRH_PASSWORD }} @@ -251,19 +214,20 @@ jobs: fi - name: Upload CDAP Standalone and CDAP DEB Bundle - # Pinned 1.11.1 version - uses: ncipollo/release-action@4c75f0f2e4ae5f3c807cf0904605408e319dcaac + # Pinned 1.14.0 version + uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5 with: allowUpdates: true omitBodyDuringUpdate: true omitNameDuringUpdate: true omitPrereleaseDuringUpdate: true + skipIfReleaseExists: ${{ env.SKIP_RELEASE }} prerelease: ${{ env.PRE_RELEASE }} removeArtifacts: ${{ env.REMOVE_ARTIFACTS }} replacesArtifacts: ${{ env.REPLACE_ARTIFACTS }} token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ env.TAG_NAME }} - body: Cask Data Appplication Platform - Release ${{ env.CDAP_VERSION }} + body: Cask Data Application Platform - Release ${{ env.CDAP_VERSION }} artifacts: | cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip,cdap-build/cdap/cdap-distributions/target/cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz @@ -279,4 +243,4 @@ jobs: - name: Report Result if: ${{ (matrix.branch == 'develop' || startsWith(matrix.branch, 'release/')) && github.event.inputs.skip_tests != 'true' && always() }} run: | - echo '{ "repository": "${{ github.repository }}", "build_name": "${{ github.workflow }}", "branch": "${{ matrix.branch }}", "conclusion": "${{ job.status }}", "started_at": "'$STARTED_AT'", "ended_at": "'$ENDED_AT'", "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", "run_id": "${{ github.run_id }}", "run_attempt": "${{ github.run_attempt }}" }' | bq insert ${{ vars.BUILD_HISTORY_TABLE }} + echo '{ "repository": "${{ github.repository }}", "build_name": "${{ github.workflow }}", "branch": "${{ matrix.branch }}", "conclusion": "${{ job.status }}", "started_at": "'$STARTED_AT'", "ended_at": "'$ENDED_AT'", "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", "run_id": "${{ github.run_id }}", "run_attempt": "${{ github.run_attempt }}" }' | bq insert ${{ vars.BUILD_HISTORY_TABLE }} \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 3d06e5e4..c04cf4d7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "cdap"] path = cdap url = ../cdap.git - branch = release/6.10 + branch = itn_messaging_fix [submodule "app-artifacts/hydrator-plugins"] path = app-artifacts/hydrator-plugins url = ../hydrator-plugins.git