-
Notifications
You must be signed in to change notification settings - Fork 16
303 lines (290 loc) · 17.6 KB
/
build.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
name: Build
# Run this workflow every time a new commit pushed to your repository
on:
push
env:
DOTNET_NOLOGO: 1
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
jobs:
# Set the job key. The key is displayed as the job name
# when a job name is not provided
tests_db:
name: Run Database Tests
runs-on: windows-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Populate Databases.yaml
shell: bash
run: |
cat > ./Tools/rdmp/Databases.yaml << EOF
CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true;
DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true;
EOF
- name: Build
run: |
dotnet build --configuration Release --verbosity minimal
- name: Install MS SQL 2019 Express LocalDB
uses: crazy-max/ghaction-chocolatey@v3
with:
args: install -r sqllocaldb --no-progress
- name: Initialise LocalDB
shell: bash
run: |
SqlLocalDB.exe create MSSQLLocalDB -s
sqlcmd -l 180 -S '(localdb)\MSSQLLocalDB' -Q "SELECT @@VERSION;"
sed -i'' -e 's/localhost/\(localdb\)\\MSSQLLocalDB/' Tests.Common/TestDatabases.txt
- uses: shogo82148/actions-setup-mysql@v1
with:
mysql-version: '8.0'
root-password: 'YourStrong!Passw0rd'
auto-start: true
- name: Build
run: dotnet build --configuration Release --verbosity minimal
- name: Initialise RDMP
run: |
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 "(localdb)\MSSQLLocalDB" TEST_ -e
- name: Create MySql Logging, DQE and Cohort Building Cache Db
run: |
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver DQE "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_dqe" --dir ~/rdmp/rdmp-yaml/
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver CohortIdentificationQueryCachingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_cache" --dir ~/rdmp/rdmp-yaml/
- name: Run integration test scripts
run: |
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_list_destroy_catalogue.yaml
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_cohort.yaml
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_dataload.yaml
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/orphan_extractable_column.yaml
- name: Test (DB)
shell: bash
run: |
rm -rf coverage
curl -L "https://dl.min.io/server/minio/release/windows-amd64/minio.exe" > minio.exe
./minio.exe server ./minio --console-address :9001 &
mkdir ~/.aws
echo -e "[minio]\naws_access_key_id=minioadmin\naws_secret_access_key=minioadmin\naws_endpoint_url=http://127.0.0.1:9001" > ~/.aws/credentials
dotnet test Rdmp.UI.Tests/Rdmp.UI.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release -e AWS_ENDPOINT_URL="http://127.0.0.1:9000" --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov
mv `find coverage -type f` db-ui.lcov
dotnet test Rdmp.Core.Tests/Rdmp.Core.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release -e AWS_ENDPOINT_URL="http://127.0.0.1:9000" --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov
mv `find coverage -type f` db-core.lcov
- uses: coverallsapp/[email protected]
with:
github-token: ${{ secrets.github_token }}
files: ./db-ui.lcov ./db-core.lcov
flag-name: unit tests
tests_file_system:
name: Run File System Tests
runs-on: windows-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Populate Databases.yaml
shell: bash
run: |
cat > ./Tools/rdmp/Databases.yaml << EOF
CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true;
DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true;
EOF
- name: Build
run: |
dotnet build --configuration Release --verbosity minimal
- name: Install MS SQL 2019 Express LocalDB
uses: crazy-max/ghaction-chocolatey@v3
with:
args: install -r sqllocaldb --no-progress
- name: Initialise LocalDB
shell: bash
run: |
SqlLocalDB.exe create MSSQLLocalDB -s
sqlcmd -l 180 -S '(localdb)\MSSQLLocalDB' -Q "SELECT @@VERSION;"
sed -i'' -e 's/localhost/\(localdb\)\\MSSQLLocalDB/' Tests.Common/TestDatabases.txt
- uses: shogo82148/actions-setup-mysql@v1
with:
mysql-version: '8.0'
root-password: 'YourStrong!Passw0rd'
auto-start: true
- name: Build
run: dotnet build --configuration Release --verbosity minimal
- name: Initialise RDMP
run: |
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 "(localdb)\MSSQLLocalDB" TEST_ -e
- name: Create MySql Logging, DQE and Cohort Building Cache Db
run: |
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver DQE "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_dqe" --dir ~/rdmp/rdmp-yaml/
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver CohortIdentificationQueryCachingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_cache" --dir ~/rdmp/rdmp-yaml/
- name: Run integration test scripts
run: |
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_list_destroy_catalogue.yaml
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_cohort.yaml
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_dataload.yaml
dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/orphan_extractable_column.yaml
- name: Test with local file system
shell: bash
run: |
echo "UseFileSystemRepo: true" >> Tests.Common/TestDatabases.txt
curl -L "https://dl.min.io/server/minio/release/windows-amd64/minio.exe" > minio.exe
./minio.exe server ./minio --console-address :9001 &
mkdir ~/.aws
echo -e "[minio]\naws_access_key_id=minioadmin\naws_secret_access_key=minioadmin\naws_endpoint_url=http:127.0.0.1:9001" > ~/.aws/credentials
dotnet test Rdmp.UI.Tests/Rdmp.UI.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release -e AWS_ENDPOINT_URL="http://127.0.0.1:9000" --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov
mv `find coverage -type f` fs-ui.lcov
dotnet test Rdmp.Core.Tests/Rdmp.Core.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release -e AWS_ENDPOINT_URL="http://127.0.0.1:9000" --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov
mv `find coverage -type f` fs-core.lcov
- uses: coverallsapp/[email protected]
with:
github-token: ${{ secrets.github_token }}
files: ./fs-ui.lcov ./fs-core.lcov
flag-name: unit tests
bundle:
name: Bundle Build
runs-on: windows-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Determine RDMP build version
id: version
shell: bash
run: perl -ne "print \"rdmpversion=\$1\n\" if /AssemblyInformationalVersion\(\"([0-9a-z.-]+)\"\)/;" SharedAssemblyInfo.cs >> $GITHUB_OUTPUT
- name: Setup .NET Core
uses: actions/[email protected]
with:
dotnet-version: 7.0.x
- name: BundleSource
shell: bash
run: |
mkdir -p Tools/BundleUpSourceIntoZip/output
rm -f Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip
echo "dir /s/b *.cs *.xml > srcbitsa.txt" | cmd
perl -pe '$_=reverse' < srcbitsa.txt | sort -t'\' -k1,1 -u | perl -pe '$_=reverse' > srcbits.txt
echo 7z a -mx=9 Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip @srcbits.txt | cmd
- name: Package
run: |
dotnet publish Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj -r win-x64 --self-contained -c Release -o PublishWinForms -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:IncludeNativeLibrariesForSelfExtract=true --verbosity minimal --nologo
dotnet publish Tools/rdmp/rdmp.csproj -r win-x64 --self-contained -c Release -o PublishWindows -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal -p:IncludeNativeLibrariesForSelfExtract=true --nologo
dotnet publish Tools/rdmp/rdmp.csproj -r linux-x64 --self-contained -c Release -o PublishLinux -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo
- name: Temporary Bundle of ddls for auto-updater
shell: bash
run: |
cp -r Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/runtimes ./PublishWinForms
cp -r Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/x64 ./PublishWinForms
cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/D3DCompiler_47_cor3.dll ./PublishWinForms
cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/mongocrypt.dll ./PublishWinForms
cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/PenImc_cor3.dll ./PublishWinForms
cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/PresentationNative_cor3.dll ./PublishWinForms
cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/vcruntime140_cor3.dll ./PublishWinForms
cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/WebView2Loader.dll ./PublishWinForms
cp Application/ResearchDataManagementPlatform/bin/Release/net8.0-windows/win-x64/wpfgfx_cor3.dll ./PublishWinForms
- name: Install Plugins
shell: bash
run: |
for plugin in https://api.github.com/repos/SMI/RdmpDicom/releases/latest https://api.github.com/repos/HICServices/HicPlugin/releases/latest https://api.github.com/repos/HICServices/RdmpExtensions/releases/latest
do
PluginName="$(cut -d/ -f6 <<< $plugin)"
NAME="$(curl -s $plugin | grep "browser_download_url.*$PluginName.*rdmp" | cut -d : -f 2,3 | cut -d "\"" -f 2)"
curl -OL $NAME
done
for platform in PublishWindows PublishLinux PublishWinForms
do
cp *.rdmp $platform
done
rm *.rdmp
- name: Sign & zip
shell: bash
run: |
dotnet tool install --global AzureSignTool
AzureSignTool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v PublishWindows/rdmp.exe
AzureSignTool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v PublishWinForms/ResearchDataManagementPlatform.exe
mkdir -p dist
(cd PublishWindows ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-win-x64.zip . | cmd)
(cd PublishLinux ; echo 7z a -mx=0 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.zip . | cmd)
mv PublishLinux rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux
echo 7z a dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux | cmd
(cd PublishWinForms ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-client.zip . | cmd)
- name: Install Perl dependencies
uses: shogo82148/[email protected]
with:
install-modules-with: cpanm
install-modules: Archive::Zip Archive::Tar
- name: Fix execute permissions
if: contains(github.ref, 'refs/tags/v') || contains('refs/heads/main refs/heads/develop',github.ref)
shell: perl {0}
run: |
use strict;
use warnings;
use Archive::Zip qw( :ERROR_CODES :CONSTANTS );
use Archive::Tar;
my ($tarname,$zipname)=('dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar','dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.zip');
# Find rdmp in .tar, set permissions 0755
my $tar=Archive::Tar->new($tarname) || die "read $tarname:$!\n";
my @tarbits=$tar->get_files();
foreach my $bit (@tarbits) {
$bit->chmod('0755') if $bit->name =~ /\/rdmp$/;
}
$tar->write( $tarname );
# Find rdmp in .zip, set permissions 0755
my $srczip = Archive::Zip->new($zipname);
my $zip=Archive::Zip->new();
foreach my $bit ($srczip->members()) {
print "Adding ",$bit->fileName(),"\n";
my $m=$zip->addString($bit->contents(),$bit->fileName());
$m->fileName($bit->fileName());
my $iszip=$bit->fileName() =~ /\.zip$/i;
$m->desiredCompressionMethod($iszip ? COMPRESSION_STORED : COMPRESSION_DEFLATED);
$m->desiredCompressionLevel($iszip ? 0 : 9 );
}
$zip->memberNamed('rdmp')->unixFileAttributes( 0755 );
$zip->overwriteAs($zipname);
- name: Compress tar
if: contains(github.ref, 'refs/tags/v') || contains('refs/heads/main refs/heads/develop',github.ref)
run: |
7z a -txz dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar.xz dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar
rm dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar
- name: Build Nuget packages
if: contains(github.ref, 'refs/tags/v')
shell: bash
run: |
for i in Rdmp.Core/Rdmp.Core.csproj Rdmp.UI/Rdmp.UI.csproj Tests.Common/Tests.Common.csproj
do
dotnet pack $i -c Release --include-symbols --nologo -o . -v:m -p:Version=${{ steps.version.outputs.rdmpversion }}
done
- name: Calculate SHA256SUMS
if: contains(github.ref, 'refs/tags/v')
run: '&{foreach ($i in Get-ChildItem dist -Exclude *SUMS|Get-FileHash) { echo "$($i.Hash) $(echo $i | Split-Path -Leaf)" }} > dist/SHA256SUMS'
- name: Archive production artifacts
uses: actions/upload-artifact@v4
with:
name: dist
path: |
dist
- name: Cache Build
id: cache-build
uses: actions/cache/save@v4
with:
path: ${{ github.workspace }}/
key: ${{ github.sha }}-your-cache-key-bundled
production-upload:
name: Production Upload
runs-on: windows-latest
needs: ['bundle','tests_db','tests_file_system']
if: contains(github.ref, 'refs/tags/v')
steps:
- uses: actions/cache/restore@v4
id: restore-build
with:
path: ${{ github.workspace }}/
key: ${{ github.sha }}-your-cache-key-bundled
- name: Determine RDMP build version
id: version
shell: bash
run: perl -ne "print \"rdmpversion=\$1\n\" if /AssemblyInformationalVersion\(\"([0-9a-z.-]+)\"\)/;" SharedAssemblyInfo.cs >> $GITHUB_OUTPUT
- name: Upload Nuget packages
run: dotnet nuget push HIC.RDMP.Plugin*${{ steps.version.outputs.rdmpversion }}.nupkg -s https://api.nuget.org/v3/index.json --skip-duplicate -k ${{ secrets.NUGET_KEY }}
- name: Upload binaries to release
uses: svenstaro/[email protected]
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: dist/*
tag: ${{ github.ref }}
overwrite: true
file_glob: true