-
Notifications
You must be signed in to change notification settings - Fork 6
354 lines (292 loc) · 12.1 KB
/
workflow.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
name: linting-and-tests
on: [pull_request]
defaults:
run:
shell: bash -l {0}
jobs:
flake8:
timeout-minutes: 40
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Setup mamba
uses: ./.github/actions/mamba
- name: Flake8
run: |
micromamba run -n modyn flake8 --version
micromamba run -n modyn flake8 modyn --statistics
mypy-typechecking:
timeout-minutes: 20
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Setup mamba
uses: ./.github/actions/mamba
- name: Mypy
run: |
micromamba run -n modyn mypy --version
micromamba run -n modyn mypy modyn
pylint:
timeout-minutes: 20
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Setup mamba
uses: ./.github/actions/mamba
- name: Pylint
run: |
micromamba run -n modyn pylint --version
micromamba run -n modyn pylint modyn
isort:
timeout-minutes: 20
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Setup mamba
uses: ./.github/actions/mamba
- name: Isort
run: |
micromamba run -n modyn isort --version
micromamba run -n modyn isort modyn --check --diff
micromamba run -n modyn isort integrationtests --check --diff
micromamba run -n modyn isort benchmark --check --diff
black:
timeout-minutes: 20
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Setup mamba
uses: ./.github/actions/mamba
- name: Black
run: |
micromamba run -n modyn black --version
micromamba run -n modyn black --check modyn --verbose --config black.toml
unittests:
timeout-minutes: 20
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Setup mamba
uses: ./.github/actions/mamba
- name: Pytest
run: |
micromamba run -n modyn pytest modyn --cov-reset --cache-clear --cov-fail-under=90
micromamba run -n modyn pytest > pytest-coverage.txt
- name: Comment coverage
uses: coroo/[email protected]
format:
timeout-minutes: 20
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: DoozyX/[email protected]
with:
source: 'modyn/storage/src modyn/storage/include modyn/storage/test'
extensions: 'hpp,cpp'
clangFormatVersion: 16
tidy:
timeout-minutes: 30
runs-on: ubuntu-latest
env:
CLANG_TIDY: clang-tidy-15
RUN_CLANG_TIDY: run-clang-tidy-15
steps:
- uses: actions/checkout@v2
- name: Install clang-tidy
run: |
sudo apt update
sudo apt install -y clang-tidy-15
cmake --version
- name: Configure CMake
working-directory: ${{github.workspace}}/modyn/storage
run: bash scripts/clang-tidy.sh build
- name: Run clang-tidy
working-directory: ${{github.workspace}}/modyn/storage
run: bash scripts/clang-tidy.sh run_tidy
cpp_build_and_test:
name: Build + Test (C++)
runs-on: ubuntu-latest
timeout-minutes: 45
outputs:
line-coverage: ${{steps.run_test_with_coverage.outputs.LINE_COVERAGE}}
branch-coverage: ${{steps.run_test_with_coverage.outputs.BRANCH_COVERAGE}}
strategy:
fail-fast: false
matrix:
build-type: [ Release, Debug ]
compiler:
- { c: gcc, cxx: g++, version: 11 }
- { c: gcc, cxx: g++, version: 12 }
- { c: clang, cxx: clang++, version: 14 }
- { c: clang, cxx: clang++, version: 16, coverage: true }
include:
# Currently, there is a linking error with zlib if we use clang 16 for sanitizers
# Let's investigate this when clang 16 is os default - one problem could be the external clang installation
- compiler: {c: clang, cxx: clang++, version: 14}
build-type: Tsan
- compiler: {c: clang, cxx: clang++, version: 14}
build-type: Asan
env:
CC: ${{matrix.compiler.c}}-${{matrix.compiler.version}}
CXX: ${{matrix.compiler.cxx}}-${{matrix.compiler.version}}
CCACHE_BASEDIR: ${{github.workspace}}/modyn/storage
steps:
- uses: actions/checkout@v2
- name: Install ccache
run: |
sudo apt update
sudo apt install -y ccache
- name: Install clang version
if: ${{ matrix.compiler.version > 14 }}
uses: KyleMayes/install-llvm-action@v1
with:
version: ${{ matrix.compiler.version }}.0
env: true
- name: Create Build Environment
run: cmake -E make_directory ${{github.workspace}}/modyn/storage/build
- name: Configure CMake
shell: bash
working-directory: ${{github.workspace}}/modyn/storage/build
# fdebug-prefix-map is for ccache to not have absolute paths interfere with caching, see https://ccache.dev/manual/3.6.html#_compiling_in_different_directories
run: >
cmake ${{github.workspace}}/modyn/storage
-DCMAKE_BUILD_TYPE=${{matrix.build-type}}
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-DCMAKE_CXX_FLAGS="-fdebug-prefix-map=${{github.workspace}}/build=."
-DMODYNSTORAGE_BUILD_PLAYGROUND=ON
-DMODYNSTORAGE_BUILD_TESTS=ON
-DMODYNSTORAGE_TEST_COVERAGE=${{matrix.compiler.coverage && 'ON' || 'OFF'}}
- name: Build
working-directory: ${{github.workspace}}/modyn/storage/build
shell: bash
run: cmake --build . --target modynstorage modynstorage-test modyn-storage playground --config ${{matrix.build-type}} -- -j8
- name: Run tests
timeout-minutes: 20
working-directory: ${{github.workspace}}/modyn/storage/build/test
shell: bash
env: {"TSAN_OPTIONS": "halt_on_error=1", "UBSAN_OPTIONS": "print_stacktrace=1:halt_on_error=1"}
run: ./modynstorage-test
# The next two steps are solely related to creating coverage reports and will only run coverage in the compiler matrix is set to true
- name: Create Coverage Report
if: ${{ matrix.compiler.coverage && matrix.build-type == 'Debug' }}
working-directory: ${{github.workspace}}/modyn/storage/build/test
run: |
llvm-profdata-14 merge -sparse default.profraw -o tests.profdata
llvm-cov-14 report -instr-profile tests.profdata -object modynstorage-test -ignore-filename-regex="build\/" -ignore-filename-regex="\/test\/" -show-region-summary=false | tail -1 | sed 's/%//g' | tr -s " " > output.txt
llvm-cov-14 show -instr-profile tests.profdata -object modynstorage-test -format=html -output-dir=coverage -ignore-filename-regex="build\/" -ignore-filename-regex="\/test\/" -show-region-summary=false
echo ::set-output name=LINE_COVERAGE::"$(cat output.txt | cut -d ' ' -f 7)"
echo ::set-output name=BRANCH_COVERAGE::"$(cat output.txt | cut -d ' ' -f 10)"
id: run_test_with_coverage
- name: Upload HTML coverage report
if: ${{ matrix.compiler.coverage && matrix.build-type == 'Debug' }}
uses: actions/upload-artifact@v2
with:
name: coverage-results
path: ${{github.workspace}}/modyn/storage/build/test/coverage
cpp_coverage_main:
name: C++ Test Coverage (gets coverage of main branch, currently not main branch because no C++ on main)
runs-on: ubuntu-latest
timeout-minutes: 45
env:
CC: clang-16
CXX: clang++-16
outputs:
line-coverage: ${{steps.run_main_test_with_coverage.outputs.LINE_COVERAGE}}
branch-coverage: ${{steps.run_main_test_with_coverage.outputs.BRANCH_COVERAGE}}
steps:
- uses: actions/checkout@v2
#with: TODO(MaxiBoether): add after merge.
# ref: main
- name: Install clang 16
uses: KyleMayes/install-llvm-action@v1
with:
version: 16.0
env: true
- name: Create Build Environment
run: |
cmake -E make_directory ${{github.workspace}}/modyn/storage/build
- name: Configure CMake
shell: bash
working-directory: ${{github.workspace}}/modyn/storage/build
run: >
cmake ${{github.workspace}}/modyn/storage -DCMAKE_BUILD_TYPE=Debug
-DMODYNSTORAGE_BUILD_PLAYGROUND=ON -DMODYNSTORAGE_BUILD_TESTS=ON -DMODYNSTORAGE_TEST_COVERAGE=ON
- name: Build
working-directory: ${{github.workspace}}/modyn/storage/build
shell: bash
run: cmake --build . --config Debug --target modynstorage modynstorage-test -- -j8
- name: Run tests
working-directory: ${{github.workspace}}/modyn/storage/build/test
shell: bash
run: ./modynstorage-test
- name: Create Coverage Report for main branch
working-directory: ${{github.workspace}}/modyn/storage/build/test
run: |
llvm-profdata-14 merge -sparse default.profraw -o tests.profdata
llvm-cov-14 report -instr-profile tests.profdata -object modynstorage-test -ignore-filename-regex="build\/" -ignore-filename-regex="\/test\/" -show-region-summary=false | tail -1 | sed 's/%//g' | tr -s " " > output.txt
echo ::set-output name=LINE_COVERAGE::"$(cat output.txt | cut -d ' ' -f 7)"
echo ::set-output name=BRANCH_COVERAGE::"$(cat output.txt | cut -d ' ' -f 10)"
id: run_main_test_with_coverage
cpp_comment_on_pr:
if: github.event.pull_request.head.repo.full_name == github.repository
runs-on: self-hosted
name: Comment Coverage Results
needs: [ cpp_build_and_test, cpp_coverage_main ]
timeout-minutes: 5
steps:
- name: Calculate changes
shell: bash
run: |
echo ::set-output name=line-changes::"$(awk 'BEGIN {printf "%+.2f", ${{ needs.cpp_build_and_test.outputs.line-coverage }}-${{ needs.cpp_coverage_main.outputs.line-coverage }}; exit}')"
echo ::set-output name=branch-changes::"$(awk 'BEGIN {printf "%+.2f", ${{ needs.cpp_build_and_test.outputs.branch-coverage }}-${{ needs.cpp_coverage_main.outputs.branch-coverage }}; exit}')"
id: calculation
- name: Comment on PR
uses: marocchino/sticky-pull-request-comment@v2
with:
message: |
 <sup>(${{ steps.calculation.outputs.line-changes }} % to main)</sup>
 <sup>(${{ steps.calculation.outputs.branch-changes }} % to main)</sup>
### Integration Tests ###
# We have them in the same workflow because it's impossible to have a simple "if workflow A runs through completely, then workflow B should run" pipeline on Github currently
# Checks whether the base container works correctly.
dockerized-unittests:
timeout-minutes: 60
runs-on: ubuntu-latest
needs:
- flake8
- mypy-typechecking
- pylint
- unittests
- isort
- black
- cpp_build_and_test
- tidy
- format
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Setup base container
uses: ./.github/actions/base
- name: Setup dev-requirements and run pytest within container
run: docker run modynbase mamba run -n modyn bash -c "pip install -r dev-requirements.txt && echo Running pytest && pytest"
integrationtests:
timeout-minutes: 60
runs-on: ubuntu-latest
needs:
- flake8
- mypy-typechecking
- pylint
- unittests
- isort
- black
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Start docker compose and exit when tests run through
run: bash run_integrationtests.sh