Disused usings; make S3 uploads less blocking #6160
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| # Run this workflow every time a new commit pushed to your repository | |
| on: | |
| push | |
| env: | |
| DOTNET_NOLOGO: 1 | |
| DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true | |
| jobs: | |
| # Set the job key. The key is displayed as the job name | |
| # when a job name is not provided | |
| tests_db: | |
| name: Run Database Tests | |
| runs-on: windows-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Populate Databases.yaml | |
| shell: bash | |
| run: | | |
| cat > ./Tools/rdmp/Databases.yaml << EOF | |
| CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; | |
| DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true; | |
| EOF | |
| - name: Build | |
| run: | | |
| dotnet build --configuration Release --verbosity minimal | |
| - name: Install MS SQL 2019 Express LocalDB | |
| uses: crazy-max/ghaction-chocolatey@v3 | |
| with: | |
| args: install -r sqllocaldb --no-progress | |
| - name: Initialise LocalDB | |
| shell: bash | |
| run: | | |
| SqlLocalDB.exe create MSSQLLocalDB -s | |
| sqlcmd -l 180 -S '(localdb)\MSSQLLocalDB' -Q "SELECT @@VERSION;" | |
| sed -i'' -e 's/localhost/\(localdb\)\\MSSQLLocalDB/' Tests.Common/TestDatabases.txt | |
| - uses: shogo82148/actions-setup-mysql@v1 | |
| with: | |
| mysql-version: '8.0' | |
| root-password: 'YourStrong!Passw0rd' | |
| auto-start: true | |
| - name: Build | |
| run: dotnet build --configuration Release --verbosity minimal | |
| - name: Initialise RDMP | |
| run: | | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 "(localdb)\MSSQLLocalDB" TEST_ -e | |
| - name: Create MySql Logging, DQE and Cohort Building Cache Db | |
| run: | | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/ | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver DQE "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_dqe" --dir ~/rdmp/rdmp-yaml/ | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver CohortIdentificationQueryCachingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_cache" --dir ~/rdmp/rdmp-yaml/ | |
| - name: Run integration test scripts | |
| run: | | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_list_destroy_catalogue.yaml | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_cohort.yaml | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_dataload.yaml | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/orphan_extractable_column.yaml | |
| - name: Test (DB) | |
| shell: bash | |
| run: | | |
| rm -rf coverage | |
| curl -L "https://dl.min.io/server/minio/release/windows-amd64/minio.exe" > minio.exe | |
| ./minio.exe server ./minio --console-address :9001 & | |
| mkdir ~/.aws | |
| echo -e "[minio]\naws_access_key_id=minioadmin\naws_secret_access_key=minioadmin\naws_endpoint_url=http://127.0.0.1:9001" > ~/.aws/credentials | |
| dotnet test Rdmp.UI.Tests/Rdmp.UI.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release -e AWS_ENDPOINT_URL="http://127.0.0.1:9000" --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov | |
| mv `find coverage -type f` db-ui.lcov | |
| dotnet test Rdmp.Core.Tests/Rdmp.Core.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release -e AWS_ENDPOINT_URL="http://127.0.0.1:9000" --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov | |
| mv `find coverage -type f` db-core.lcov | |
| - uses: coverallsapp/[email protected] | |
| with: | |
| github-token: ${{ secrets.github_token }} | |
| files: ./db-ui.lcov ./db-core.lcov | |
| flag-name: unit tests | |
| tests_file_system: | |
| name: Run File System Tests | |
| runs-on: windows-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Populate Databases.yaml | |
| shell: bash | |
| run: | | |
| cat > ./Tools/rdmp/Databases.yaml << EOF | |
| CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; | |
| DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true; | |
| EOF | |
| - name: Build | |
| run: | | |
| dotnet build --configuration Release --verbosity minimal | |
| - name: Install MS SQL 2019 Express LocalDB | |
| uses: crazy-max/ghaction-chocolatey@v3 | |
| with: | |
| args: install -r sqllocaldb --no-progress | |
| - name: Initialise LocalDB | |
| shell: bash | |
| run: | | |
| SqlLocalDB.exe create MSSQLLocalDB -s | |
| sqlcmd -l 180 -S '(localdb)\MSSQLLocalDB' -Q "SELECT @@VERSION;" | |
| sed -i'' -e 's/localhost/\(localdb\)\\MSSQLLocalDB/' Tests.Common/TestDatabases.txt | |
| - uses: shogo82148/actions-setup-mysql@v1 | |
| with: | |
| mysql-version: '8.0' | |
| root-password: 'YourStrong!Passw0rd' | |
| auto-start: true | |
| - name: Build | |
| run: dotnet build --configuration Release --verbosity minimal | |
| - name: Initialise RDMP | |
| run: | | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 "(localdb)\MSSQLLocalDB" TEST_ -e | |
| - name: Create MySql Logging, DQE and Cohort Building Cache Db | |
| run: | | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/ | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver DQE "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_dqe" --dir ~/rdmp/rdmp-yaml/ | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver CohortIdentificationQueryCachingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_cache" --dir ~/rdmp/rdmp-yaml/ | |
| - name: Run integration test scripts | |
| run: | | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_list_destroy_catalogue.yaml | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_cohort.yaml | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_dataload.yaml | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/orphan_extractable_column.yaml | |
| - name: Test with local file system | |
| shell: bash | |
| run: | | |
| echo "UseFileSystemRepo: true" >> Tests.Common/TestDatabases.txt | |
| curl -L "https://dl.min.io/server/minio/release/windows-amd64/minio.exe" > minio.exe | |
| ./minio.exe server ./minio --console-address :9001 & | |
| mkdir ~/.aws | |
| echo -e "[minio]\naws_access_key_id=minioadmin\naws_secret_access_key=minioadmin\naws_endpoint_url=http:127.0.0.1:9001" > ~/.aws/credentials | |
| dotnet test Rdmp.UI.Tests/Rdmp.UI.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release -e AWS_ENDPOINT_URL="http://127.0.0.1:9000" --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov | |
| mv `find coverage -type f` fs-ui.lcov | |
| dotnet test Rdmp.Core.Tests/Rdmp.Core.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release -e AWS_ENDPOINT_URL="http://127.0.0.1:9000" --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov | |
| mv `find coverage -type f` fs-core.lcov | |
| - uses: coverallsapp/[email protected] | |
| with: | |
| github-token: ${{ secrets.github_token }} | |
| files: ./fs-ui.lcov ./fs-core.lcov | |
| flag-name: unit tests | |
| bundle: | |
| name: Bundle Build | |
| runs-on: windows-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Determine RDMP build version | |
| id: version | |
| shell: bash | |
| run: perl -ne "print \"rdmpversion=\$1\n\" if /AssemblyInformationalVersion\(\"([0-9a-z.-]+)\"\)/;" SharedAssemblyInfo.cs >> $GITHUB_OUTPUT | |
| - name: Setup .NET Core | |
| uses: actions/[email protected] | |
| with: | |
| dotnet-version: 7.0.x | |
| - name: BundleSource | |
| shell: bash | |
| run: | | |
| mkdir -p Tools/BundleUpSourceIntoZip/output | |
| rm -f Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip | |
| echo "dir /s/b *.cs *.xml > srcbitsa.txt" | cmd | |
| perl -pe '$_=reverse' < srcbitsa.txt | sort -t'\' -k1,1 -u | perl -pe '$_=reverse' > srcbits.txt | |
| echo 7z a -mx=9 Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip @srcbits.txt | cmd | |
| - name: Package | |
| run: | | |
| dotnet publish Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj -r win-x64 --self-contained -c Release -o PublishWinForms -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:IncludeNativeLibrariesForSelfExtract=true --verbosity minimal --nologo | |
| dotnet publish Tools/rdmp/rdmp.csproj -r win-x64 --self-contained -c Release -o PublishWindows -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal -p:IncludeNativeLibrariesForSelfExtract=true --nologo | |
| dotnet publish Tools/rdmp/rdmp.csproj -r linux-x64 --self-contained -c Release -o PublishLinux -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo | |
| - name: Temporary Bundle of ddls for auto-updater | |
| shell: bash | |
| run: | | |
| cp -r Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/runtimes ./PublishWinForms | |
| cp -r Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/x64 ./PublishWinForms | |
| cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/D3DCompiler_47_cor3.dll ./PublishWinForms | |
| cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/PenImc_cor3.dll ./PublishWinForms | |
| cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/PresentationNative_cor3.dll ./PublishWinForms | |
| cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/vcruntime140_cor3.dll ./PublishWinForms | |
| cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/WebView2Loader.dll ./PublishWinForms | |
| cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/wpfgfx_cor3.dll ./PublishWinForms | |
| - name: Install Plugins | |
| shell: bash | |
| run: | | |
| for plugin in https://api.github.com/repos/SMI/RdmpDicom/releases/latest https://api.github.com/repos/HICServices/HicPlugin/releases/latest https://api.github.com/repos/HICServices/RdmpExtensions/releases/latest | |
| do | |
| PluginName="$(cut -d/ -f6 <<< $plugin)" | |
| NAME="$(curl -s $plugin | grep "browser_download_url.*$PluginName.*rdmp" | cut -d : -f 2,3 | cut -d "\"" -f 2)" | |
| curl -OL $NAME | |
| done | |
| for platform in PublishWindows PublishLinux PublishWinForms | |
| do | |
| cp *.rdmp $platform | |
| done | |
| rm *.rdmp | |
| - name: Sign & zip | |
| shell: bash | |
| run: | | |
| dotnet tool install --global AzureSignTool | |
| AzureSignTool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v PublishWindows/rdmp.exe | |
| AzureSignTool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v PublishWinForms/ResearchDataManagementPlatform.exe | |
| mkdir -p dist | |
| (cd PublishWindows ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-win-x64.zip . | cmd) | |
| (cd PublishLinux ; echo 7z a -mx=0 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.zip . | cmd) | |
| mv PublishLinux rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux | |
| echo 7z a dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux | cmd | |
| (cd PublishWinForms ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-client.zip . | cmd) | |
| - name: Install Perl dependencies | |
| uses: shogo82148/[email protected] | |
| with: | |
| install-modules-with: cpanm | |
| install-modules: Archive::Zip Archive::Tar | |
| - name: Fix execute permissions | |
| if: contains(github.ref, 'refs/tags/v') || contains('refs/heads/main refs/heads/develop',github.ref) | |
| shell: perl {0} | |
| run: | | |
| use strict; | |
| use warnings; | |
| use Archive::Zip qw( :ERROR_CODES :CONSTANTS ); | |
| use Archive::Tar; | |
| my ($tarname,$zipname)=('dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar','dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.zip'); | |
| # Find rdmp in .tar, set permissions 0755 | |
| my $tar=Archive::Tar->new($tarname) || die "read $tarname:$!\n"; | |
| my @tarbits=$tar->get_files(); | |
| foreach my $bit (@tarbits) { | |
| $bit->chmod('0755') if $bit->name =~ /\/rdmp$/; | |
| } | |
| $tar->write( $tarname ); | |
| # Find rdmp in .zip, set permissions 0755 | |
| my $srczip = Archive::Zip->new($zipname); | |
| my $zip=Archive::Zip->new(); | |
| foreach my $bit ($srczip->members()) { | |
| print "Adding ",$bit->fileName(),"\n"; | |
| my $m=$zip->addString($bit->contents(),$bit->fileName()); | |
| $m->fileName($bit->fileName()); | |
| my $iszip=$bit->fileName() =~ /\.zip$/i; | |
| $m->desiredCompressionMethod($iszip ? COMPRESSION_STORED : COMPRESSION_DEFLATED); | |
| $m->desiredCompressionLevel($iszip ? 0 : 9 ); | |
| } | |
| $zip->memberNamed('rdmp')->unixFileAttributes( 0755 ); | |
| $zip->overwriteAs($zipname); | |
| - name: Compress tar | |
| if: contains(github.ref, 'refs/tags/v') || contains('refs/heads/main refs/heads/develop',github.ref) | |
| run: | | |
| 7z a -txz dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar.xz dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar | |
| rm dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar | |
| - name: Build Nuget packages | |
| if: contains(github.ref, 'refs/tags/v') | |
| shell: bash | |
| run: | | |
| for i in Rdmp.Core/Rdmp.Core.csproj Rdmp.UI/Rdmp.UI.csproj Tests.Common/Tests.Common.csproj | |
| do | |
| dotnet pack $i -c Release --include-symbols --nologo -o . -v:m -p:Version=${{ steps.version.outputs.rdmpversion }} | |
| done | |
| - name: Calculate SHA256SUMS | |
| if: contains(github.ref, 'refs/tags/v') | |
| run: '&{foreach ($i in Get-ChildItem dist -Exclude *SUMS|Get-FileHash) { echo "$($i.Hash) $(echo $i | Split-Path -Leaf)" }} > dist/SHA256SUMS' | |
| - name: Archive production artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dist | |
| path: | | |
| dist | |
| - name: Cache Build | |
| id: cache-build | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ${{ github.workspace }}/ | |
| key: ${{ github.sha }}-your-cache-key-bundled | |
| production-upload: | |
| name: Production Upload | |
| runs-on: windows-latest | |
| needs: ['bundle','tests_db','tests_file_system'] | |
| if: contains(github.ref, 'refs/tags/v') | |
| steps: | |
| - uses: actions/cache/restore@v4 | |
| id: restore-build | |
| with: | |
| path: ${{ github.workspace }}/ | |
| key: ${{ github.sha }}-your-cache-key-bundled | |
| - name: Determine RDMP build version | |
| id: version | |
| shell: bash | |
| run: perl -ne "print \"rdmpversion=\$1\n\" if /AssemblyInformationalVersion\(\"([0-9a-z.-]+)\"\)/;" SharedAssemblyInfo.cs >> $GITHUB_OUTPUT | |
| - name: Upload Nuget packages | |
| run: dotnet nuget push HIC.RDMP.Plugin*${{ steps.version.outputs.rdmpversion }}.nupkg -s https://api.nuget.org/v3/index.json --skip-duplicate -k ${{ secrets.NUGET_KEY }} | |
| - name: Upload binaries to release | |
| uses: svenstaro/[email protected] | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| file: dist/* | |
| tag: ${{ github.ref }} | |
| overwrite: true | |
| file_glob: true |