integration-tests.yml

# Heavyweight integration tests
name: Integration tests

on:
  push:
    branches:
      - main
    paths-ignore:
      - docs/
  pull_request:
    branches:
      - main
    paths-ignore:
      - docs/
  workflow_dispatch:

permissions:
  id-token: write # This is required for requesting the JWT
  contents: read  # This is required for actions/checkout

  
jobs:
  set-version-number:
    name: Set version number
    runs-on: ubuntu-latest
    outputs:
      nuGetVersion: ${{ steps.gitversion.outputs.nuGetVersionV2 }}
      semVer: ${{ steps.gitversion.outputs.fullSemVer }}
      is-release: ${{ steps.gitversion.outputs.CommitsSinceVersionSource == 0 }}
      #is-release: 'true'
    
    steps:
    - name: Setup .NET 9
      uses: actions/setup-dotnet@v4
      with:
        dotnet-version: 9.0.x
    - name: Checkout
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Install GitVersion
      uses: gittools/actions/gitversion/setup@v3.1.3
      with:
        versionSpec: '5.x'
    - name: Determine Version
      id: gitversion
      uses: gittools/actions/gitversion/execute@v3.1.3

  build-standalone:
    name: Build cli
    needs: set-version-number

    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: 
          # - win-x64
          # - win-x86
          # - win-arm64
          # - linux-musl-x64
          # - linux-musl-arm64
          - linux-x64
          # - linux-arm64
          # - osx-x64
    steps:
    - uses: actions/checkout@v4
    - name: Setup .NET 9
      uses: actions/setup-dotnet@v4
      with:
        dotnet-version: 9.0.x

    - name: Publish self-contained ${{ matrix.arch }}
      run: dotnet publish ./src/grate/grate.csproj -r ${{ matrix.arch }} -c release --self-contained -p:SelfContained=true -o ./publish/${{ matrix.arch }}/self-contained
      env:
        VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }}
        TargetFramework: net9.0

    - name: Upload self-contained ${{ matrix.arch }}
      uses: actions/upload-artifact@v4
      with:
        name: grate-${{ matrix.arch }}-self-contained-${{ needs.set-version-number.outputs.nuGetVersion }}
        path: ./publish/${{ matrix.arch }}/self-contained/*

  build-standalone-mac-arm64:
    if: false
    name: Build cli
    needs: set-version-number

    # Use macos-14 to build osx-arm64, it runs on M1, see 
    # https://github.blog/changelog/2024-01-30-github-actions-introducing-the-new-m1-macos-runner-available-to-open-source/
    #
    # I've earlier had problems with that the trimmed, self-contained binary for osx-arm64 that was built on Linux
    # did not work when opened on an actual mac with arm64.

    runs-on: macos-14
    strategy:
      matrix:
        arch: [ "osx-arm64" ]

    steps:
    - uses: actions/checkout@v4
    - name: Setup .NET 9
      uses: actions/setup-dotnet@v4
      with:
        dotnet-version: 9.0.x

    - name: Publish self-contained ${{ matrix.arch }}
      run: dotnet publish ./src/grate/grate.csproj -r ${{ matrix.arch }} -c release --self-contained -p:SelfContained=true -o ./publish/${{ matrix.arch }}/self-contained
      env:
        VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }}
        TargetFramework: net9.0

    - name: Upload self-contained ${{ matrix.arch }}
      uses: actions/upload-artifact@v4
      with:
        name: grate-${{ matrix.arch }}-self-contained-${{ needs.set-version-number.outputs.nuGetVersion }}
        path: ./publish/${{ matrix.arch }}/self-contained/*

  build-tests:
    name: Build tests
    needs: set-version-number

    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: 
          #- win-x64
          - linux-x64
          #- osx-x64
    steps:
    - uses: actions/checkout@v4
    - name: Setup .NET 9
      uses: actions/setup-dotnet@v4
      with:
         dotnet-version: |
          6.0.x
          7.0.x
          8.0.x
          9.0.x

    - name: Build Test DLLs ${{ matrix.arch }}
      run: |
        dotnet restore -r ${{ matrix.arch }} unittests/UnitTests.slnf
        dotnet publish -r ${{ matrix.arch }} -c release --no-restore ./unittests/UnitTests.slnf -o ./integration-tests/${{ matrix.arch }}

      env:
        VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }}

    - name: Upload Test DLLs ${{ matrix.arch }}
      uses: actions/upload-artifact@v4
      with:
        name: integration-tests-${{ matrix.arch }}-${{ needs.set-version-number.outputs.nuGetVersion }}
        path: ./integration-tests/${{ matrix.arch }}/*

  build-tests-mac-arm64:
    if: false
    name: Build tests
    needs: set-version-number

    runs-on: macos-14
    strategy:
      matrix:
        arch: [ "osx-arm64" ]

    steps:
    - uses: actions/checkout@v4
    - name: Setup .NET 8 & 9
      uses: actions/setup-dotnet@v4
      with:
        dotnet-version: |
          8.0.x
          9.0.x

    - name: Build Test DLLs ${{ matrix.arch }}
      run: |
        dotnet restore -r ${{ matrix.arch }} unittests/UnitTests.slnf
        dotnet publish -r ${{ matrix.arch }} -c release --no-restore ./unittests/UnitTests.slnf -o ./integration-tests/${{ matrix.arch }}

      env:
        VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }}

    - name: Upload Test DLLs ${{ matrix.arch }}
      uses: actions/upload-artifact@v4
      with:
        name: integration-tests-${{ matrix.arch }}-${{ needs.set-version-number.outputs.nuGetVersion }}
        path: ./integration-tests/${{ matrix.arch }}/*

  setup-test-environment:
    if: false
    name: Set up test enviroment
    environment: integration-tests
    runs-on: ubuntu-latest
    env:
      tf_actions_working_dir: "${{ github.workspace }}/test-infra/terraform/test-environment"
      ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      
    defaults:
      run:
        working-directory: ${{ env.tf_actions_working_dir }}

    outputs:
      # Need to split the connection string in two, and assemble when used, as GitHub Actions refuses to 
      # write anything to the outputs that have what it considers sensitive values. And, 'Pwd=<password>'
      # apparently is.
      mariadb_connection_string: ${{ steps.show.outputs.mariadb_connection_string }}
      mariadb_pw: ${{ steps.show.outputs.mariadb_pw }}
      dbs: ${{ steps.show.outputs.dbs }}
      #encoded_dbs: ${{ steps.show.outputs.encoded_dbs }}

    steps:
      - uses: actions/checkout@v4

      - name: Log into Azure
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      - name: Show account info
        run: >
          az account show

      - uses: hashicorp/setup-terraform@v3

      - run: terraform init

      - name: Terraform Validate
        run: terraform validate

      - name: Terraform plan
        id: plan
        run: terraform plan -input=false

      # - run: echo ${{ steps.plan.outputs.stdout }}
      # - run: echo ${{ steps.plan.outputs.stderr }}
      # - run: echo ${{ steps.plan.outputs.exitcode }}
  
      - name: Exit on invalid Terraform plan
        if: steps.plan.outcome == 'failure'
        run: exit 1

      - name: Terraform Apply
        id: apply
        run: terraform apply -auto-approve

      - name: Get terraform outputs
        id: show
        run: |
          terraform show -json > state.json

          # Have to base64 encode, to avoid github seeing a pw, and refuse to output at all
          DBS=$(cat state.json | jq -c '[ .values.outputs.dbs.value[] | .connectionstring |= @base64 ] ' )

          {
            echo 'dbs<<EOFAAA'
            echo ${DBS}
            echo EOFAAA
          } >> "$GITHUB_OUTPUT"

  docker-integration-tests:
    name: ${{ matrix.database.name }}
    continue-on-error: true
    timeout-minutes: 25

    needs:
      - set-version-number
      - build-standalone
      - build-tests

    strategy:
      fail-fast: false
      matrix:
        database:
          # - name: "SqlServer 2022"
          #   type: SqlServer
          #   image: "mcr.microsoft.com/mssql/server:2022-latest"
          - name: "SqlServer 2022 (Ubuntu 22.04)"
            type: SqlServer
            image: "mcr.microsoft.com/mssql/server:2022-preview-ubuntu-22.04"
          - name: "SqlServer 2019" 
            type: SqlServer
            image: "mcr.microsoft.com/mssql/server:2019-CU26-ubuntu-20.04"
          - name: "SqlServer 2017" 
            type: SqlServer
            image: "mcr.microsoft.com/mssql/server:2017-latest"
          - name: "PostgreSQL 16" 
            type: PostgreSQL
            image: "postgres:16"
          - name: "PostgreSQL 15" 
            type: PostgreSQL
            image: "postgres:15"
          - name: "PostgreSQL 14" 
            type: PostgreSQL
            image: "postgres:14"
          - name: "PostgreSQL 13" 
            type: PostgreSQL
            image: "postgres:13"
          - name: "PostgreSQL 12" 
            type: PostgreSQL
            image: "postgres:12"
          - name: "MariaDB 10.10" 
            type: MariaDB
            image: "mariadb:10.10"
          - name: "Sqlite" 
            type: Sqlite
            image: "irrelevant"
          - name: "Oracle Free latest" 
            type: Oracle
            image: "gvenzl/oracle-free:latest-faststart"
          - name: "Oracle XE latest" 
            type: Oracle
            image: "gvenzl/oracle-xe:latest-faststart"
          - name: "Oracle XE 18" 
            type: Oracle
            image: "gvenzl/oracle-xe:18-faststart"


    runs-on: ubuntu-latest
    steps:
    - name: Test run info
      shell: pwsh
      run: |
        echo "Test run info:"
        echo "===================="
        echo "Name: ${{ matrix.database.name }}"
        echo "Database type: ${{ matrix.database.type }}"
        echo "Image: ${{ matrix.database.image }}"

    - uses: actions/checkout@v4

    - uses: actions/download-artifact@v4
      name: Download executables
      with:
        name: grate-linux-x64-self-contained-${{ needs.set-version-number.outputs.nuGetVersion }}
        path: executables/linux-x64

    - uses: actions/download-artifact@v4
      name: Download test DLLs
      with:
        name: integration-tests-linux-x64-${{ needs.set-version-number.outputs.nuGetVersion }}
        path: integration-tests/linux-x64

    - name: Setup .NET 9
      uses: actions/setup-dotnet@v4
      with:
        dotnet-version: 9.0.x

    - name: chmod u+x
      run: chmod u+x $GrateExecutablePath
      env:
        GrateExecutablePath: ${{ github.workspace }}/executables/linux-x64/grate

    - name: Docker Test CmdLine
      if: ${{ !cancelled() }}
      shell: pwsh
      run: |
        dotnet test integration-tests/linux-x64/CommandLine.${{ matrix.database.type }}.dll `
        --logger:"xunit;Environment=${{ matrix.database.name }} (docker);LogFilePath=/tmp/test-results/docker/CommandLine.${{ matrix.database.name }}.xml"
      env:
        LogLevel: Information
        GrateExecutablePath: ${{ github.workspace }}/executables/linux-x64/grate
        TZ: UTC
        GrateTestConfig__DockerImage: ${{ matrix.database.image }} 

    - name: Docker Test Docker
      # TODO: We need to integrate build of a docker image to test into the pipeline, and push it to the local registry,
      # and use that one, to run the integration tests. If not, we are testing the latest published released.
      #if: ${{ !cancelled() }}
      if: false
      shell: pwsh
      run: |
        dotnet test integration-tests/linux-x64/Docker.${{ matrix.database.type }}.dll `
        --logger:"xunit;Environment=${{ matrix.database.name }} (docker);LogFilePath=/tmp/test-results/docker/Docker.${{ matrix.database.name }}.xml"
      env:
        LogLevel: Information
        TZ: UTC
        GrateTestConfig__DockerImage: ${{ matrix.database.image }} 

    - name: Upload Unit Test Results
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: "docker ${{ matrix.database.name }} XML test results"
        path: |
          /tmp/test-results/docker/CommandLine.${{ matrix.database.name }}.xml
          /tmp/test-results/docker/Docker.${{ matrix.database.name }}.xml
        retention-days: 1

  integration-test:
    if: false
    environment: integration-tests
    name: ${{ matrix.database.name }}, ${{ matrix.os.name }}
    needs:
      - set-version-number
      - setup-test-environment
      - build-standalone
      - build-standalone-mac-arm64
      - build-tests
      - build-tests-mac-arm64
    continue-on-error: true
    timeout-minutes: 25

    strategy:
      fail-fast: false
      # Try to keep this the same as the number of databases we test
      # (this will avoid many test suites against the same database server simultaneously)
      max-parallel: 12
      matrix:
        os:
          - name: windows-latest
            arch: win-x64
            executable: grate.exe
          - name: ubuntu-latest
            arch: linux-x64
            executable: grate
          - name: macos-latest
            arch: osx-x64
            executable: grate
          # macos-14 is M1 (arm64)
          - name: macos-14
            arch: osx-arm64
            executable: grate
        database: ${{ fromJson(needs.setup-test-environment.outputs.dbs) }}

    runs-on: ${{ matrix.os.name }}
    steps:
    - name: Test run info
      shell: pwsh
      run: |
        echo "Test run info:"
        echo "===================="
        echo "OS: ${{ matrix.os.name }}"
        echo "Arch: ${{ matrix.os.arch }}"
        echo "Executable: ${{ matrix.os.executable }}"
        echo "Database: ${{ matrix.database.name }}"
        echo "Cleaned database name: ${{ matrix.database.cleaned_name }}"

        echo "Connection string: $([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String( "${{ matrix.database.connectionstring }}" )) )"

    - uses: actions/checkout@v4

    - uses: actions/download-artifact@v4
      name: Download executables
      with:
        name: grate-${{ matrix.os.arch }}-self-contained-${{ needs.set-version-number.outputs.nuGetVersion }}
        path: executables/${{ matrix.os.arch }}

    - uses: actions/download-artifact@v4
      name: Download test DLLs
      with:
        name: integration-tests-${{ matrix.os.arch }}-${{ needs.set-version-number.outputs.nuGetVersion }}
        path: integration-tests/${{ matrix.os.arch }}

    - name: Setup .NET 9
      uses: actions/setup-dotnet@v4
      with:
        dotnet-version: 9.0.x

    - name: chmod u+x
      run: chmod u+x $GrateExecutablePath
      if: ${{ matrix.os.arch != 'win-x64' }}
      env:
        GrateExecutablePath: ${{ github.workspace }}/executables/${{ matrix.os.arch }}/${{ matrix.os.executable }}

    - name: Log into Azure
      uses: azure/login@v2
      with:
        client-id: ${{ secrets.AZURE_CLIENT_ID }}
        tenant-id: ${{ secrets.AZURE_TENANT_ID }}
        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

    - name: Test lib
      shell: pwsh
      # Theses are not stable enought to run against an external database. Disable for now.
      if: false
      run: |
        $env:GrateTestConfig__AdminConnectionString="$([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String( "${{ matrix.database.connectionstring }}" )) )";

        dotnet test integration-tests/${{ matrix.os.arch }}/${{ matrix.database.type }}.dll `
        --logger:"xunit;Environment=${{ matrix.os.arch }},${{ matrix.database.name }};LogFilePath=/tmp/test-results/${{ matrix.os.arch }}/${{ matrix.database.cleaned_name }}.xml" -- `
        -MaxCpuCount 2

      env:
        LogLevel: Critical
        TZ: UTC

    - name: Test CmdLine
      if: ${{ !cancelled() }}
      shell: pwsh
      run: |
        $env:GrateTestConfig__AdminConnectionString="$([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String( "${{ matrix.database.connectionstring }}" )) )";

        dotnet test integration-tests/${{ matrix.os.arch }}/CommandLine.${{ matrix.database.type }}.dll `
        --logger:"xunit;Environment=${{ matrix.os.arch }},${{ matrix.database.name }};LogFilePath=/tmp/test-results/${{ matrix.os.arch }}/CommandLine.${{ matrix.database.cleaned_name }}.xml" -- `
        -MaxCpuCount 2

      env:
        LogLevel: Information
        GrateExecutablePath: ${{ github.workspace }}/executables/${{ matrix.os.arch }}/${{ matrix.os.executable }}
        TZ: UTC

    - name: Upload Unit Test Results
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: "${{ matrix.os.arch }} ${{ matrix.database.cleaned_name }} XML test results"
        path: |
          /tmp/test-results/${{ matrix.os.arch }}/${{ matrix.database.cleaned_name }}.xml
          /tmp/test-results/${{ matrix.os.arch }}/CommandLine.${{ matrix.database.cleaned_name }}.xml
        retention-days: 1

    # - name: Fail build if not successful
    #   id: fail-if-failed
    #   run: |
    #      [ "${FAILURE}" = "true" ] && echo "Failures found. Marking as failed" && exit 1;
    #      [ "${CANCELLED}" = "true" ] && echo "Job cancelled. Marking as failed" && exit 1;
    #   if: always()
    #   #if: contains(steps.*.result, 'failure') || contains(steps.*.result, 'cancelled')
    #   env:
    #     FAILURE: ${{ contains(steps.*.result, 'failure') }}
    #     CANCELLED: ${{ contains(steps.*.result, 'cancelled') }}

  report:
    runs-on: ubuntu-latest
    name: Create test report
    #needs: integration-test
    needs: docker-integration-tests
    if: always()

    steps:
      - name: Download XML test reports
        uses: actions/download-artifact@v4
        with:
          path: test-results

      - name: Install smink
        run: |
          BASEURL="https://github.com/erikbra/smink/releases/download/"
          VERSION="0.4.0"

          INSTALL_DIR="/tmp/smink"
          SMINK="${INSTALL_DIR}/smink"

          FILENAME="smink-linux-x64-${VERSION}.zip"
          FULL_URL="${BASEURL}${VERSION}/${FILENAME}"
          DOWNLOAD_DIR="/tmp/smink-download"

          DOWNLOAD_FILE="${DOWNLOAD_DIR}/${FILENAME}"

          test -f "${SMINK}" && \
          echo "smink already installed - not installing" || \
          echo "Installing smink v ${VERSION}" && \
          (test -d "${DOWNLOAD_DIR}" || mkdir -p "${DOWNLOAD_DIR}") && \
          (test -d "${INSTALL_DIR}" || mkdir -p "${INSTALL_DIR}") && \
          (test -f "${DOWNLOAD_FILE}" || curl -o "${DOWNLOAD_FILE}" -sL "${FULL_URL}") && \
          unzip -o "${DOWNLOAD_FILE}" -d "${INSTALL_DIR}" 

          chmod u+x "${SMINK}"

      - name: Create HTML test report
        run: |

          XML_DIR='/tmp/xml'
          REPORT_DIR='/tmp/test-results'

          test -d $XML_DIR  || mkdir $XML_DIR
          find "test-results" -name '*.xml' -exec cp "{}" $XML_DIR \;
          echo "Found the following test results files:"
          ls -1 $XML_DIR

          test -d $REPORT_DIR || mkdir $REPORT_DIR
          echo "Running smink"
          /tmp/smink/smink "${XML_DIR}/*.xml" "${REPORT_DIR}/grate integration tests $(date +'%Y-%m-%d %H.%M.%S %z').html" --title "grate integration tests"

      - name: Upload HTML test report
        uses: actions/upload-artifact@v4
        with:
          name: HTML test report
          path: |
            /tmp/test-results/*.html
          retention-days: 10
      
  teardown-test-environment:
    name: Tear down test enviroment
    environment: integration-tests
    runs-on: ubuntu-latest
    needs: integration-test
    #if: always()
    if: false
    env:
      tf_actions_working_dir: "${{ github.workspace }}/test-infra/terraform/test-environment"
      ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      ARM_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      ARM_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      
    defaults:
      run:
        working-directory: ${{ env.tf_actions_working_dir }}

    steps:
      - uses: actions/checkout@v4

      - name: Log into Azure
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      - name: Show account info
        run: >
          az account show

      - uses: hashicorp/setup-terraform@v3

      - run: terraform init

      - name: Terraform Validate
        run: terraform validate

      - id: plan-destroy
        run: terraform plan -destroy -input=false

      - name: Terraform Destroy
        run: terraform destroy -auto-approve

  # fail-if-failed:
  #   name: Fail build on failure
  #   if: always()
  #   needs:
  #     - integration-test
  #     - teardown-test-environment
  #   runs-on: ubuntu-latest
  #   steps:
  #     - name: Fail build if not successful
  #       id: fail-if-failed
  #       run: |
  #         [ "${FAILURE}" = "true" ] && echo "Failures found. Marking as failed" && exit 1;
  #         [ "${CANCELLED}" = "true" ] && echo "Job cancelled. Marking as failed" && exit 1;
  #       if: always()
  #       #if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
  #       env:
  #         FAILURE: ${{ contains(needs.*.result, 'failure') }}
  #         CANCELLED: ${{ contains(needs.*.result, 'cancelled') }}