Integration tests #212
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Heavyweight integration tests | |
| name: Integration tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - docs/ | |
| workflow_dispatch: | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| contents: read # This is required for actions/checkout | |
| jobs: | |
| set-version-number: | |
| name: Set version number | |
| runs-on: ubuntu-latest | |
| outputs: | |
| nuGetVersion: ${{ steps.gitversion.outputs.nuGetVersionV2 }} | |
| semVer: ${{ steps.gitversion.outputs.fullSemVer }} | |
| is-release: ${{ steps.gitversion.outputs.CommitsSinceVersionSource == 0 }} | |
| #is-release: 'true' | |
| steps: | |
| - name: Setup .NET 8 | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: 8.0.x | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install GitVersion | |
| uses: gittools/actions/gitversion/[email protected] | |
| with: | |
| versionSpec: '5.x' | |
| - name: Determine Version | |
| id: gitversion | |
| uses: gittools/actions/gitversion/[email protected] | |
| build-standalone: | |
| name: Build cli | |
| needs: set-version-number | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| arch: | |
| - win-x64 | |
| # - win-x86 | |
| # - win-arm64 | |
| # - linux-musl-x64 | |
| # - linux-musl-arm64 | |
| - linux-x64 | |
| # - linux-arm64 | |
| - osx-x64 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup .NET 8 | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: 8.0.x | |
| - name: Publish self-contained ${{ matrix.arch }} | |
| run: dotnet publish ./src/grate/grate.csproj -f net8.0 -r ${{ matrix.arch }} -c release --self-contained -p:SelfContained=true -o ./publish/${{ matrix.arch }}/self-contained | |
| env: | |
| VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }} | |
| - name: Upload self-contained ${{ matrix.arch }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: grate-${{ matrix.arch }}-self-contained-${{ needs.set-version-number.outputs.nuGetVersion }} | |
| path: ./publish/${{ matrix.arch }}/self-contained/* | |
| build-standalone-mac-arm64: | |
| name: Build cli | |
| needs: set-version-number | |
| # Use macos-14 to build osx-arm64, it runs on M1, see | |
| # https://github.blog/changelog/2024-01-30-github-actions-introducing-the-new-m1-macos-runner-available-to-open-source/ | |
| # | |
| # I've earlier had problems with that the trimmed, self-contained binary for osx-arm64 that was built on Linux | |
| # did not work when opened on an actual mac with arm64. | |
| runs-on: macos-14 | |
| strategy: | |
| matrix: | |
| arch: [ "osx-arm64" ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup .NET 8 | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: 8.0.x | |
| - name: Publish self-contained ${{ matrix.arch }} | |
| run: dotnet publish ./src/grate/grate.csproj -f net8.0 -r ${{ matrix.arch }} -c release --self-contained -p:SelfContained=true -o ./publish/${{ matrix.arch }}/self-contained | |
| env: | |
| VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }} | |
| - name: Upload self-contained ${{ matrix.arch }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: grate-${{ matrix.arch }}-self-contained-${{ needs.set-version-number.outputs.nuGetVersion }} | |
| path: ./publish/${{ matrix.arch }}/self-contained/* | |
| build-tests: | |
| name: Build tests | |
| needs: set-version-number | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| arch: | |
| - win-x64 | |
| - linux-x64 | |
| - osx-x64 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup .NET 8 | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: 8.0.x | |
| - name: Build Test DLLs ${{ matrix.arch }} | |
| run: | | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.MariaDB -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.SqlServer -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.Oracle -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.PostgreSQL -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.Sqlite -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/MariaDB -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/SqlServer -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/Oracle -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/PostgreSQL -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/Sqlite -o ./integration-tests/${{ matrix.arch }} | |
| env: | |
| VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }} | |
| - name: Upload Test DLLs ${{ matrix.arch }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: integration-tests-${{ matrix.arch }}-${{ needs.set-version-number.outputs.nuGetVersion }} | |
| path: ./integration-tests/${{ matrix.arch }}/* | |
| build-tests-mac-arm64: | |
| name: Build tests | |
| needs: set-version-number | |
| runs-on: macos-14 | |
| strategy: | |
| matrix: | |
| arch: [ "osx-arm64" ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup .NET 8 | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: 8.0.x | |
| - name: Build Test DLLs ${{ matrix.arch }} | |
| run: | | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.MariaDB -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.SqlServer -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.Oracle -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.PostgreSQL -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.Sqlite -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/MariaDB -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/SqlServer -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/Oracle -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/PostgreSQL -o ./integration-tests/${{ matrix.arch }} | |
| dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/Sqlite -o ./integration-tests/${{ matrix.arch }} | |
| env: | |
| VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }} | |
| - name: Upload Test DLLs ${{ matrix.arch }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: integration-tests-${{ matrix.arch }}-${{ needs.set-version-number.outputs.nuGetVersion }} | |
| path: ./integration-tests/${{ matrix.arch }}/* | |
| setup-test-environment: | |
| #if: false | |
| name: Set up test enviroment | |
| environment: integration-tests | |
| runs-on: ubuntu-latest | |
| env: | |
| tf_actions_working_dir: "${{ github.workspace }}/test-infra/terraform/test-environment" | |
| ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| defaults: | |
| run: | |
| working-directory: ${{ env.tf_actions_working_dir }} | |
| outputs: | |
| # Need to split the connection string in two, and assemble when used, as GitHub Actions refuses to | |
| # write anything to the outputs that have what it considers sensitive values. And, 'Pwd=<password>' | |
| # apparently is. | |
| mariadb_connection_string: ${{ steps.show.outputs.mariadb_connection_string }} | |
| mariadb_pw: ${{ steps.show.outputs.mariadb_pw }} | |
| dbs: ${{ steps.show.outputs.dbs }} | |
| #encoded_dbs: ${{ steps.show.outputs.encoded_dbs }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Log into Azure | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: Show account info | |
| run: > | |
| az account show | |
| - uses: hashicorp/setup-terraform@v3 | |
| - run: terraform init | |
| - name: Terraform Validate | |
| run: terraform validate | |
| - name: Terraform plan | |
| id: plan | |
| run: terraform plan -input=false | |
| # - run: echo ${{ steps.plan.outputs.stdout }} | |
| # - run: echo ${{ steps.plan.outputs.stderr }} | |
| # - run: echo ${{ steps.plan.outputs.exitcode }} | |
| - name: Exit on invalid Terraform plan | |
| if: steps.plan.outcome == 'failure' | |
| run: exit 1 | |
| - name: Terraform Apply | |
| id: apply | |
| run: terraform apply -auto-approve | |
| - name: Get terraform outputs | |
| id: show | |
| run: | | |
| terraform show -json > state.json | |
| # Have to base64 encode, to avoid github seeing a pw, and refuse to output at all | |
| DBS=$(cat state.json | jq -c '[ .values.outputs.dbs.value[] | .connectionstring |= @base64 ] ' ) | |
| { | |
| echo 'dbs<<EOFAAA' | |
| echo ${DBS} | |
| echo EOFAAA | |
| } >> "$GITHUB_OUTPUT" | |
| # read-outputs: | |
| # environment: integration-tests | |
| # name: "Print outputs" | |
| # needs: | |
| # - setup-test-environment | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - name: Output matrix variable | |
| # run: | | |
| # echo "Output variables:" | |
| # echo "====================" | |
| # echo "All outputs: ${{ needs.setup-test-environment.outputs }}" | |
| # echo "Databases: ${{ needs.setup-test-environment.outputs.dbs }}" | |
| integration-test: | |
| environment: integration-tests | |
| name: ${{ matrix.database.name }}, ${{ matrix.os.name }} | |
| needs: | |
| - set-version-number | |
| - setup-test-environment | |
| - build-standalone | |
| - build-standalone-mac-arm64 | |
| - build-tests | |
| - build-tests-mac-arm64 | |
| continue-on-error: true | |
| timeout-minutes: 25 | |
| strategy: | |
| fail-fast: false | |
| # Try to keep this the same as the number of databases we test | |
| # (this will avoid many test suites against the same database server simultaneously) | |
| max-parallel: 12 | |
| matrix: | |
| os: | |
| # - name: windows-latest | |
| # arch: win-x64 | |
| # executable: grate.exe | |
| - name: ubuntu-latest | |
| arch: linux-x64 | |
| executable: grate | |
| # - name: macos-latest | |
| # arch: osx-x64 | |
| # executable: grate | |
| # macos-14 is M1 (arm64) | |
| # - name: macos-14 | |
| # arch: osx-arm64 | |
| # executable: grate | |
| database: ${{ fromJson(needs.setup-test-environment.outputs.dbs) }} | |
| runs-on: ${{ matrix.os.name }} | |
| steps: | |
| - name: Test run info | |
| shell: pwsh | |
| run: | | |
| echo "Test run info:" | |
| echo "====================" | |
| echo "OS: ${{ matrix.os.name }}" | |
| echo "Arch: ${{ matrix.os.arch }}" | |
| echo "Executable: ${{ matrix.os.executable }}" | |
| echo "Database: ${{ matrix.database.name }}" | |
| echo "Cleaned database name: ${{ matrix.database.cleaned_name }}" | |
| echo "Connection string: $([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String( "${{ matrix.database.connectionstring }}" )) )" | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| name: Download executables | |
| with: | |
| name: grate-${{ matrix.os.arch }}-self-contained-${{ needs.set-version-number.outputs.nuGetVersion }} | |
| path: executables/${{ matrix.os.arch }} | |
| - uses: actions/download-artifact@v4 | |
| name: Download test DLLs | |
| with: | |
| name: integration-tests-${{ matrix.os.arch }}-${{ needs.set-version-number.outputs.nuGetVersion }} | |
| path: integration-tests/${{ matrix.os.arch }} | |
| - name: Setup .NET 8 | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: 8.0.x | |
| - name: chmod u+x | |
| run: chmod u+x $GrateExecutablePath | |
| if: ${{ matrix.os.arch != 'win-x64' }} | |
| env: | |
| GrateExecutablePath: ${{ github.workspace }}/executables/${{ matrix.os.arch }}/${{ matrix.os.executable }} | |
| - name: Log into Azure | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: Test lib | |
| shell: pwsh | |
| #timeout-minutes: 12 | |
| run: | | |
| $env:GrateTestConfig__AdminConnectionString="$([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String( "${{ matrix.database.connectionstring }}" )) )"; | |
| dotnet test integration-tests/${{ matrix.os.arch }}/${{ matrix.database.type }}.dll ` | |
| --logger:"xunit;Environment=${{ matrix.os.arch }},${{ matrix.database.name }};LogFilePath=/tmp/test-results/${{ matrix.os.arch }}/${{ matrix.database.cleaned_name }}.xml" -- ` | |
| -MaxCpuCount 2 | |
| env: | |
| LogLevel: Critical | |
| TZ: UTC | |
| - name: Test CmdLine | |
| #timeout-minutes: 6 | |
| if: ${{ !cancelled() }} | |
| shell: pwsh | |
| run: | | |
| $env:GrateTestConfig__AdminConnectionString="$([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String( "${{ matrix.database.connectionstring }}" )) )"; | |
| dotnet test integration-tests/${{ matrix.os.arch }}/CommandLine.${{ matrix.database.type }}.dll ` | |
| --logger:"xunit;Environment=${{ matrix.os.arch }},${{ matrix.database.name }};LogFilePath=/tmp/test-results/${{ matrix.os.arch }}/CommandLine.${{ matrix.database.cleaned_name }}.xml" -- ` | |
| -MaxCpuCount 2 | |
| env: | |
| LogLevel: Information | |
| GrateExecutablePath: ${{ github.workspace }}/executables/${{ matrix.os.arch }}/${{ matrix.os.executable }} | |
| TZ: UTC | |
| - name: Upload Unit Test Results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: "${{ matrix.os.arch }} ${{ matrix.database.cleaned_name }} XML test results" | |
| path: | | |
| /tmp/test-results/${{ matrix.os.arch }}/${{ matrix.database.cleaned_name }}.xml | |
| /tmp/test-results/${{ matrix.os.arch }}/CommandLine.${{ matrix.database.cleaned_name }}.xml | |
| retention-days: 1 | |
| # - name: Fail build if not successful | |
| # id: fail-if-failed | |
| # run: | | |
| # [ "${FAILURE}" = "true" ] && echo "Failures found. Marking as failed" && exit 1; | |
| # [ "${CANCELLED}" = "true" ] && echo "Job cancelled. Marking as failed" && exit 1; | |
| # if: always() | |
| # #if: contains(steps.*.result, 'failure') || contains(steps.*.result, 'cancelled') | |
| # env: | |
| # FAILURE: ${{ contains(steps.*.result, 'failure') }} | |
| # CANCELLED: ${{ contains(steps.*.result, 'cancelled') }} | |
| report: | |
| runs-on: ubuntu-latest | |
| name: Create test report | |
| needs: integration-test | |
| if: always() | |
| steps: | |
| - name: Download XML test reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: test-results | |
| - name: Install smink | |
| run: | | |
| BASEURL="https://github.com/erikbra/smink/releases/download/" | |
| VERSION="0.4.0" | |
| INSTALL_DIR="/tmp/smink" | |
| SMINK="${INSTALL_DIR}/smink" | |
| FILENAME="smink-linux-x64-${VERSION}.zip" | |
| FULL_URL="${BASEURL}${VERSION}/${FILENAME}" | |
| DOWNLOAD_DIR="/tmp/smink-download" | |
| DOWNLOAD_FILE="${DOWNLOAD_DIR}/${FILENAME}" | |
| test -f "${SMINK}" && \ | |
| echo "smink already installed - not installing" || \ | |
| echo "Installing smink v ${VERSION}" && \ | |
| (test -d "${DOWNLOAD_DIR}" || mkdir -p "${DOWNLOAD_DIR}") && \ | |
| (test -d "${INSTALL_DIR}" || mkdir -p "${INSTALL_DIR}") && \ | |
| (test -f "${DOWNLOAD_FILE}" || curl -o "${DOWNLOAD_FILE}" -sL "${FULL_URL}") && \ | |
| unzip -o "${DOWNLOAD_FILE}" -d "${INSTALL_DIR}" | |
| chmod u+x "${SMINK}" | |
| - name: Create HTML test report | |
| run: | | |
| XML_DIR='/tmp/xml' | |
| REPORT_DIR='/tmp/test-results' | |
| test -d $XML_DIR || mkdir $XML_DIR | |
| find "test-results" -name '*.xml' -exec cp "{}" $XML_DIR \; | |
| echo "Found the following test results files:" | |
| ls -1 $XML_DIR | |
| test -d $REPORT_DIR || mkdir $REPORT_DIR | |
| echo "Running smink" | |
| /tmp/smink/smink "${XML_DIR}/*.xml" "${REPORT_DIR}/grate integration tests $(date +'%Y-%m-%d %H.%M.%S %z').html" --title "grate integration tests" | |
| - name: Upload HTML test report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: HTML test report | |
| path: | | |
| /tmp/test-results/*.html | |
| retention-days: 10 | |
| teardown-test-environment: | |
| name: Tear down test enviroment | |
| environment: integration-tests | |
| runs-on: ubuntu-latest | |
| needs: integration-test | |
| if: always() | |
| #if: false | |
| env: | |
| tf_actions_working_dir: "${{ github.workspace }}/test-infra/terraform/test-environment" | |
| ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| defaults: | |
| run: | |
| working-directory: ${{ env.tf_actions_working_dir }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Log into Azure | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: Show account info | |
| run: > | |
| az account show | |
| - uses: hashicorp/setup-terraform@v3 | |
| - run: terraform init | |
| - name: Terraform Validate | |
| run: terraform validate | |
| - id: plan-destroy | |
| run: terraform plan -destroy -input=false | |
| # - name: Terraform Plan Status | |
| # if: steps.plan.outcome == 'failure' | |
| # run: exit 1 | |
| - name: Terraform Destroy | |
| #run: terraform apply -destroy -auto-approve | |
| run: terraform destroy -auto-approve | |
| # fail-if-failed: | |
| # name: Fail build on failure | |
| # if: always() | |
| # needs: | |
| # - integration-test | |
| # - teardown-test-environment | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - name: Fail build if not successful | |
| # id: fail-if-failed | |
| # run: | | |
| # [ "${FAILURE}" = "true" ] && echo "Failures found. Marking as failed" && exit 1; | |
| # [ "${CANCELLED}" = "true" ] && echo "Job cancelled. Marking as failed" && exit 1; | |
| # if: always() | |
| # #if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') | |
| # env: | |
| # FAILURE: ${{ contains(needs.*.result, 'failure') }} | |
| # CANCELLED: ${{ contains(needs.*.result, 'cancelled') }} | |