Test ITN for messagin service #1607
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright © 2022 Cask Data, Inc. | |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may not | |
| # use this file except in compliance with the License. You may obtain a copy of | |
| # the License at | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | |
| # License for the specific language governing permissions and limitations under | |
| # the License. | |
| name: Build and Unit Test | |
| on: | |
| push | |
| env: | |
| TAG_NAME: latest | |
| PRE_RELEASE: true | |
| REPLACE_ARTIFACTS: true | |
| REMOVE_ARTIFACTS: true | |
| SKIP_RELEASE: false | |
| MAX_ATTEMPTS: 3 | |
| jobs: | |
| set-branch-matrix: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: set-matrix | |
| run: | | |
| if [ ${{ github.event_name }} != "workflow_dispatch" ]; | |
| then | |
| echo 'BRANCH_MATRIX={"include":[{"branch":"develop"}, {"branch":"release/6.9"}, {"branch":"release/6.10"}]}' >> $GITHUB_ENV | |
| else | |
| echo 'BRANCH_MATRIX={"include":[{"branch":"${{ github.event.inputs.branch }}"}]}' >> $GITHUB_ENV | |
| if [ "${{ github.event.inputs.run_tests_with_retries }}" == "false" ] | |
| then | |
| echo 'MAX_ATTEMPTS=1' >> $GITHUB_ENV | |
| fi | |
| fi | |
| outputs: | |
| matrix: ${{ env.BRANCH_MATRIX }} | |
| retries: ${{ env. MAX_ATTEMPTS }} | |
| build: | |
| needs: set-branch-matrix | |
| runs-on: cdapio-hub-k8-runner | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.set-branch-matrix.outputs.matrix) }} | |
| steps: | |
| - name: Get Start Time | |
| run: echo "STARTED_AT=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV | |
| - name: Get Secrets from GCP Secret Manager | |
| id: 'secrets' | |
| uses: 'google-github-actions/get-secretmanager-secrets@v2' | |
| if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} | |
| with: | |
| secrets: |- | |
| CDAP_OSSRH_USERNAME:cdapio-github-builds/CDAP_OSSRH_USERNAME | |
| CDAP_OSSRH_PASSWORD:cdapio-github-builds/CDAP_OSSRH_PASSWORD | |
| CDAP_GPG_PASSPHRASE:cdapio-github-builds/CDAP_GPG_PASSPHRASE | |
| CDAP_GPG_PRIVATE_KEY:cdapio-github-builds/CDAP_GPG_PRIVATE_KEY | |
| - name: Recursively Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| path: cdap-build | |
| ref: ${{ matrix.branch }} | |
| - name: Update Submodules | |
| working-directory: cdap-build | |
| run: | | |
| git submodule update --init --recursive --remote | |
| - name: Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-${{ github.workflow }} | |
| - name: Set up Artifact Name # Removes invalid artifact name characters: ",:,<,>,|,*,?,\,/. | |
| if: "${{ github.event.inputs.skip_tests != 'true' }}" | |
| run: | | |
| name=$(echo -n "${{ matrix.branch }}" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g') | |
| echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV | |
| - name: Build Standalone | |
| # Pinned version 2.8.2 | |
| uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482 | |
| with: | |
| timeout_minutes: 60 | |
| max_attempts: 3 | |
| retry_on: error | |
| on_retry_command: echo "Build Standalone failed in this attempt, retrying ..." | |
| command: | | |
| cd cdap-build | |
| MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 | |
| - name: Find Build Version | |
| working-directory: cdap-build/cdap | |
| run: | | |
| export VERSION=$(ls cdap-standalone/target/cdap-sandbox*zip | cut --delimiter=- --fields="-3" --complement | rev | cut --delimiter="." --fields=1 --complement | rev) | |
| echo "CDAP_VERSION=${VERSION}" | |
| echo "CDAP_VERSION=${VERSION}" >> $GITHUB_ENV | |
| if [[ ($VERSION != *-SNAPSHOT) && ("${{ github.event.inputs.keep_existing_release }}" != "false") ]]; | |
| then | |
| echo "PRE_RELEASE=false" >> $GITHUB_ENV | |
| echo "REPLACE_ARTIFACTS=false" >> $GITHUB_ENV | |
| echo "REMOVE_ARTIFACTS=false" >> $GITHUB_ENV | |
| echo "SKIP_RELEASE=true" >> $GITHUB_ENV | |
| echo "Release will not be overwritten if exists." | |
| else | |
| echo "Release will be overwritten if exists." | |
| fi | |
| - name: Upload CDAP Standalone | |
| if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} | |
| uses: actions/upload-artifact@v4 # https://github.com/actions/upload-artifact#zipped-artifact-downloads | |
| with: | |
| name: cdap-sandbox-${{env.CDAP_VERSION}}.zip | |
| path: cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip | |
| - name: Set up GPG conf | |
| if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} | |
| run: | | |
| echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf | |
| echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf | |
| - name: Import GPG key | |
| if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }} | |
| run: | | |
| echo "$GPG_PRIVATE_KEY" > private.key | |
| gpg --import --batch private.key | |
| env: | |
| GPG_PRIVATE_KEY: ${{ steps.secrets.outputs.CDAP_GPG_PRIVATE_KEY }} | |
| - name: Maven Deploy | |
| run: | | |
| cd cdap-build | |
| if [[ (${{ matrix.branch }} == "develop") || (${{ matrix.branch }} == release/*) ]]; | |
| then | |
| retry_count=0 | |
| failed_module="" | |
| while [ $retry_count -lt 3 ]; do | |
| if [ -n "$failed_module" ]; | |
| then | |
| build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE -rf :"$failed_module" 2>&1 || true) | |
| else | |
| build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE 2>&1 || true) | |
| fi | |
| echo "$build_output" | |
| if [ $(echo "$build_output" | grep -c "BUILD FAILURE") -gt 0 ]; | |
| then | |
| echo "[WARNING] Deployment failed, retrying..." | |
| failed_module=$(echo "$build_output" | grep -e 'mvn <args> -rf' | sed -n 's/.*mvn <args> -rf ://p') | |
| echo "[INFO] FAILED MODULE = $failed_module" | |
| retry_count=$((retry_count + 1)) | |
| else | |
| echo "[INFO] Deployment successful" | |
| break | |
| fi | |
| done | |
| if [ $retry_count -ge 3 ]; | |
| then | |
| echo "[ERROR] Max retries reached..., deployment failed" | |
| exit 1 | |
| fi | |
| else | |
| mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true | |
| fi | |
| env: | |
| CDAP_OSSRH_USERNAME: ${{ steps.secrets.outputs.CDAP_OSSRH_USERNAME }} | |
| CDAP_OSSRH_PASSWORD: ${{ steps.secrets.outputs.CDAP_OSSRH_PASSWORD }} | |
| CDAP_GPG_PASSPHRASE: ${{ steps.secrets.outputs.CDAP_GPG_PASSPHRASE }} | |
| MAVEN_OPTS: "-Xmx12G" | |
| - name: Build DEB Bundle | |
| working-directory: cdap-build/cdap | |
| run: | | |
| mkdir -p cdap-distributions/target/deb-bundle-tmp | |
| cd cdap-distributions/target/deb-bundle-tmp | |
| cp ../../../*/target/*.deb . | |
| tar zcf ../cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz *.deb | |
| - name: Set Up Tag | |
| working-directory: cdap-build | |
| run: | | |
| if [[ ${{ matrix.branch }} == release/* ]]; | |
| then | |
| echo "TAG_NAME=v$CDAP_VERSION" >> $GITHUB_ENV | |
| echo "TAG_NAME=v$CDAP_VERSION" | |
| git tag -f v$CDAP_VERSION | |
| git push -f origin refs/tags/v$CDAP_VERSION:refs/tags/v$CDAP_VERSION | |
| elif [[ ${{ matrix.branch }} != "develop" ]]; | |
| then | |
| export TAG=$(git check-ref-format --normalize "tags/${{ matrix.branch }}/tag" | cut -c6- | rev | cut -c5- | rev) | |
| echo "TAG_NAME=$TAG" >> $GITHUB_ENV | |
| echo "TAG_NAME=$TAG" | |
| git tag -f $TAG | |
| git push -f origin refs/tags/$TAG:refs/tags/$TAG | |
| else | |
| git tag -f ${{ env.TAG_NAME }} | |
| git push -f origin refs/tags/${{ env.TAG_NAME }}:refs/tags/${{ env.TAG_NAME }} | |
| fi | |
| - name: Upload CDAP Standalone and CDAP DEB Bundle | |
| # Pinned 1.14.0 version | |
| uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5 | |
| with: | |
| allowUpdates: true | |
| omitBodyDuringUpdate: true | |
| omitNameDuringUpdate: true | |
| omitPrereleaseDuringUpdate: true | |
| skipIfReleaseExists: ${{ env.SKIP_RELEASE }} | |
| prerelease: ${{ env.PRE_RELEASE }} | |
| removeArtifacts: ${{ env.REMOVE_ARTIFACTS }} | |
| replacesArtifacts: ${{ env.REPLACE_ARTIFACTS }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| tag: ${{ env.TAG_NAME }} | |
| body: Cask Data Application Platform - Release ${{ env.CDAP_VERSION }} | |
| artifacts: | | |
| cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip,cdap-build/cdap/cdap-distributions/target/cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz | |
| - name: Alert team if build fails | |
| if: ${{ (matrix.branch == 'develop' || startsWith(matrix.branch, 'release/')) && failure() }} | |
| run: | | |
| gcloud logging write cdapio-github-builds-logs '{ "message": "Build failure", "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' --payload-type=json --severity=ERROR | |
| - name: Get End Time | |
| if: ${{ always() }} | |
| run: echo "ENDED_AT=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV | |
| - name: Report Result | |
| if: ${{ (matrix.branch == 'develop' || startsWith(matrix.branch, 'release/')) && github.event.inputs.skip_tests != 'true' && always() }} | |
| run: | | |
| echo '{ "repository": "${{ github.repository }}", "build_name": "${{ github.workflow }}", "branch": "${{ matrix.branch }}", "conclusion": "${{ job.status }}", "started_at": "'$STARTED_AT'", "ended_at": "'$ENDED_AT'", "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", "run_id": "${{ github.run_id }}", "run_attempt": "${{ github.run_attempt }}" }' | bq insert ${{ vars.BUILD_HISTORY_TABLE }} |