From e90bbe9c1fd7c18bdedf9234121c299beb12a423 Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Fri, 17 Nov 2023 07:44:24 +0100 Subject: [PATCH 1/5] GHA: split into smaller jobs Split current C++/Python/Notebook tests into three separate workflows. Closes #2195 Closes #2014 --- .github/actions/setup-amici-cpp/action.yml | 35 +++++ .github/workflows/test_python_cplusplus.yml | 138 +++++++++++++++----- 2 files changed, 137 insertions(+), 36 deletions(-) create mode 100644 .github/actions/setup-amici-cpp/action.yml diff --git a/.github/actions/setup-amici-cpp/action.yml b/.github/actions/setup-amici-cpp/action.yml new file mode 100644 index 0000000000..21d20701d7 --- /dev/null +++ b/.github/actions/setup-amici-cpp/action.yml @@ -0,0 +1,35 @@ +name: Set up AMICI C++ +description: | + Build the AMICI C++ interface and set things for for coverage analysis. + (Currently ubuntu-only). + +runs: + using: "composite" + steps: + # BioNetGen Path + - run: echo "BNGPATH=${GITHUB_WORKSPACE}/ThirdParty/BioNetGen-2.7.0" >> $GITHUB_ENV + shell: bash + + # use all available cores + - run: echo "AMICI_PARALLEL_COMPILE=" >> $GITHUB_ENV + shell: bash + + # enable coverage + - run: echo "ENABLE_GCOV_COVERAGE=TRUE" >> $GITHUB_ENV + shell: bash + + - name: Set up Sonar tools + uses: ./.github/actions/setup-sonar-tools + + - name: Install apt dependencies + uses: ./.github/actions/install-apt-dependencies + + - name: Build AMICI dependencies + run: scripts/buildDependencies.sh + shell: bash + + - name: Build AMICI + run: scripts/buildAmici.sh + shell: bash + env: + CI_SONARCLOUD: "TRUE" diff --git a/.github/workflows/test_python_cplusplus.yml b/.github/workflows/test_python_cplusplus.yml index acc8759a3c..65e805e356 100644 --- a/.github/workflows/test_python_cplusplus.yml +++ b/.github/workflows/test_python_cplusplus.yml @@ -8,20 +8,13 @@ on: - master jobs: - build: + ubuntu-cpp-tests: name: Tests Ubuntu - - # TODO: prepare image with more deps preinstalled runs-on: ubuntu-22.04 - env: - AMICI_PARALLEL_COMPILE: "" - ENABLE_GCOV_COVERAGE: "TRUE" - CI_SONARCLOUD: "TRUE" - strategy: matrix: - python-version: [ 3.9 ] + python-version: [ "3.9" ] steps: - name: Set up Python ${{ matrix.python-version }} @@ -33,31 +26,51 @@ jobs: - run: git fetch --prune --unshallow - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV - - run: echo "BNGPATH=${GITHUB_WORKSPACE}/ThirdParty/BioNetGen-2.7.0" >> $GITHUB_ENV - - name: Set up Sonar tools - uses: ./.github/actions/setup-sonar-tools + - name: Set up AMICI C++ libraries + uses: ./.github/actions/setup-amici-cpp - - name: Install apt dependencies - uses: ./.github/actions/install-apt-dependencies + - name: C++ tests + run: scripts/run-cpp-tests.sh - # install amici dependencies - - name: apt + - name: Codecov CPP + uses: codecov/codecov-action@v3.1.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.info + flags: cpp + fail_ci_if_error: true + + - name: Run sonar-scanner + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} run: | - sudo apt-get update \ - && sudo apt-get install -y \ - cmake \ - python3-venv \ - lcov + sonar-scanner \ + -Dsonar.cfamily.build-wrapper-output=bw-output \ + -Dsonar.projectVersion="$(git describe --abbrev=4 --dirty=-dirty --always --tags | tr -d '\n')" - - name: Build AMICI dependencies - run: scripts/buildDependencies.sh + ubuntu-python-tests: + name: Tests Ubuntu + runs-on: ubuntu-22.04 - - name: Build AMICI - run: CI_SONARCLOUD=TRUE scripts/buildAmici.sh + strategy: + matrix: + python-version: [ "3.9" ] - - name: C++ tests - run: scripts/run-cpp-tests.sh + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - uses: actions/checkout@v3 + - run: git fetch --prune --unshallow + + - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV + + - name: Set up AMICI C++ libraries + uses: ./.github/actions/setup-amici-cpp - name: Install python package run: scripts/installAmiciSource.sh @@ -74,7 +87,6 @@ jobs: --durations=10 \ ${AMICI_DIR}/python/tests - - name: Python tests splines if: ${{ github.base_ref == 'master' || github.event.merge_group.base_ref == 'master'}} run: | @@ -86,6 +98,68 @@ jobs: --durations=10 \ ${AMICI_DIR}/python/tests/test_splines.py + - name: Codecov Python + uses: codecov/codecov-action@v3.1.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: build/coverage_py.xml + flags: python + fail_ci_if_error: true + verbose: true + + - name: lcov + run: | + lcov --compat-libtool --no-external \ + -d ${AMICI_DIR}/build/CMakeFiles/amici.dir/src \ + -b ${AMICI_DIR} -c -o coverage_cpp.info \ + && lcov --compat-libtool --no-external \ + -d ${AMICI_DIR}/python/sdist/build/temp_amici/CMakeFiles/amici.dir/src \ + -b ${AMICI_DIR}/python/sdist -c -o coverage_py.info \ + && lcov -a coverage_cpp.info -a coverage_py.info -o coverage.info + + - name: Codecov CPP + uses: codecov/codecov-action@v3.1.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.info + flags: cpp_python + fail_ci_if_error: true + + - name: Run sonar-scanner + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + run: | + sonar-scanner \ + -Dsonar.cfamily.build-wrapper-output=bw-output \ + -Dsonar.projectVersion="$(git describe --abbrev=4 --dirty=-dirty --always --tags | tr -d '\n')" + + + ubuntu-notebook-tests: + name: Tests Ubuntu + runs-on: ubuntu-22.04 + + strategy: + matrix: + python-version: [ "3.9" ] + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - uses: actions/checkout@v3 + - run: git fetch --prune --unshallow + + - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV + + - name: Set up AMICI C++ libraries + uses: ./.github/actions/setup-amici-cpp + + - name: Install python package + run: scripts/installAmiciSource.sh + - name: Install notebook dependencies run: | source build/venv/bin/activate \ @@ -116,14 +190,6 @@ jobs: -b ${AMICI_DIR}/python/sdist -c -o coverage_py.info \ && lcov -a coverage_cpp.info -a coverage_py.info -o coverage.info - - name: Codecov CPP - uses: codecov/codecov-action@v3.1.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: coverage.info - flags: cpp - fail_ci_if_error: true - - name: Run sonar-scanner env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From cda0afebe21c534150aaeb28e74958c41bf6e908 Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Fri, 17 Nov 2023 07:49:45 +0100 Subject: [PATCH 2/5] names --- .github/workflows/test_python_cplusplus.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_python_cplusplus.yml b/.github/workflows/test_python_cplusplus.yml index 65e805e356..1dbcc94246 100644 --- a/.github/workflows/test_python_cplusplus.yml +++ b/.github/workflows/test_python_cplusplus.yml @@ -9,7 +9,7 @@ on: jobs: ubuntu-cpp-tests: - name: Tests Ubuntu + name: C++ tests Ubuntu runs-on: ubuntu-22.04 strategy: @@ -51,7 +51,7 @@ jobs: -Dsonar.projectVersion="$(git describe --abbrev=4 --dirty=-dirty --always --tags | tr -d '\n')" ubuntu-python-tests: - name: Tests Ubuntu + name: Python tests Ubuntu runs-on: ubuntu-22.04 strategy: @@ -136,7 +136,7 @@ jobs: ubuntu-notebook-tests: - name: Tests Ubuntu + name: Notebook tests Ubuntu runs-on: ubuntu-22.04 strategy: From 0651eb28b1cf5dd1a90b6da445761d1b81f7a99b Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Fri, 17 Nov 2023 07:52:09 +0100 Subject: [PATCH 3/5] deps --- .github/actions/setup-amici-cpp/action.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/actions/setup-amici-cpp/action.yml b/.github/actions/setup-amici-cpp/action.yml index 21d20701d7..09ec9311bf 100644 --- a/.github/actions/setup-amici-cpp/action.yml +++ b/.github/actions/setup-amici-cpp/action.yml @@ -24,6 +24,15 @@ runs: - name: Install apt dependencies uses: ./.github/actions/install-apt-dependencies + - name: Install additional apt dependencies + run: | + sudo apt-get update \ + && sudo apt-get install -y \ + cmake \ + python3-venv \ + lcov + shell: bash + - name: Build AMICI dependencies run: scripts/buildDependencies.sh shell: bash From ae2c0615a0c92885143dc9042f079032ebcfa1d7 Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Fri, 17 Nov 2023 08:04:39 +0100 Subject: [PATCH 4/5] ? --- .github/workflows/test_python_cplusplus.yml | 149 -------------------- 1 file changed, 149 deletions(-) diff --git a/.github/workflows/test_python_cplusplus.yml b/.github/workflows/test_python_cplusplus.yml index 1dbcc94246..8c2ad208ab 100644 --- a/.github/workflows/test_python_cplusplus.yml +++ b/.github/workflows/test_python_cplusplus.yml @@ -50,155 +50,6 @@ jobs: -Dsonar.cfamily.build-wrapper-output=bw-output \ -Dsonar.projectVersion="$(git describe --abbrev=4 --dirty=-dirty --always --tags | tr -d '\n')" - ubuntu-python-tests: - name: Python tests Ubuntu - runs-on: ubuntu-22.04 - - strategy: - matrix: - python-version: [ "3.9" ] - - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - uses: actions/checkout@v3 - - run: git fetch --prune --unshallow - - - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV - - - name: Set up AMICI C++ libraries - uses: ./.github/actions/setup-amici-cpp - - - name: Install python package - run: scripts/installAmiciSource.sh - - - name: Python tests - run: | - source build/venv/bin/activate \ - && pytest \ - --ignore-glob=*petab* \ - --ignore-glob=*test_splines.py \ - --cov=amici \ - --cov-report=xml:"${AMICI_DIR}/build/coverage_py.xml" \ - --cov-append \ - --durations=10 \ - ${AMICI_DIR}/python/tests - - - name: Python tests splines - if: ${{ github.base_ref == 'master' || github.event.merge_group.base_ref == 'master'}} - run: | - source build/venv/bin/activate \ - && pytest \ - --cov=amici \ - --cov-report=xml:"${AMICI_DIR}/build/coverage_py.xml" \ - --cov-append \ - --durations=10 \ - ${AMICI_DIR}/python/tests/test_splines.py - - - name: Codecov Python - uses: codecov/codecov-action@v3.1.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: build/coverage_py.xml - flags: python - fail_ci_if_error: true - verbose: true - - - name: lcov - run: | - lcov --compat-libtool --no-external \ - -d ${AMICI_DIR}/build/CMakeFiles/amici.dir/src \ - -b ${AMICI_DIR} -c -o coverage_cpp.info \ - && lcov --compat-libtool --no-external \ - -d ${AMICI_DIR}/python/sdist/build/temp_amici/CMakeFiles/amici.dir/src \ - -b ${AMICI_DIR}/python/sdist -c -o coverage_py.info \ - && lcov -a coverage_cpp.info -a coverage_py.info -o coverage.info - - - name: Codecov CPP - uses: codecov/codecov-action@v3.1.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: coverage.info - flags: cpp_python - fail_ci_if_error: true - - - name: Run sonar-scanner - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: | - sonar-scanner \ - -Dsonar.cfamily.build-wrapper-output=bw-output \ - -Dsonar.projectVersion="$(git describe --abbrev=4 --dirty=-dirty --always --tags | tr -d '\n')" - - - ubuntu-notebook-tests: - name: Notebook tests Ubuntu - runs-on: ubuntu-22.04 - - strategy: - matrix: - python-version: [ "3.9" ] - - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - uses: actions/checkout@v3 - - run: git fetch --prune --unshallow - - - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV - - - name: Set up AMICI C++ libraries - uses: ./.github/actions/setup-amici-cpp - - - name: Install python package - run: scripts/installAmiciSource.sh - - - name: Install notebook dependencies - run: | - source build/venv/bin/activate \ - && pip install jax[cpu] - - - name: example notebooks - run: scripts/runNotebook.sh python/examples/example_*/ - - - name: doc notebooks - run: scripts/runNotebook.sh documentation/GettingStarted.ipynb - - - name: Codecov Python - uses: codecov/codecov-action@v3.1.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: build/coverage_py.xml - flags: python - fail_ci_if_error: true - verbose: true - - - name: lcov - run: | - lcov --compat-libtool --no-external \ - -d ${AMICI_DIR}/build/CMakeFiles/amici.dir/src \ - -b ${AMICI_DIR} -c -o coverage_cpp.info \ - && lcov --compat-libtool --no-external \ - -d ${AMICI_DIR}/python/sdist/build/temp_amici/CMakeFiles/amici.dir/src \ - -b ${AMICI_DIR}/python/sdist -c -o coverage_py.info \ - && lcov -a coverage_cpp.info -a coverage_py.info -o coverage.info - - - name: Run sonar-scanner - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: | - sonar-scanner \ - -Dsonar.cfamily.build-wrapper-output=bw-output \ - -Dsonar.projectVersion="$(git describe --abbrev=4 --dirty=-dirty --always --tags | tr -d '\n')" - osx: name: Tests OSX runs-on: macos-latest From 16d030bc34308aaf34777e8659c383d1c8e7ed4e Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Fri, 17 Nov 2023 08:06:03 +0100 Subject: [PATCH 5/5] .. --- .github/workflows/test_python_cplusplus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_python_cplusplus.yml b/.github/workflows/test_python_cplusplus.yml index 8c2ad208ab..e82adc34f4 100644 --- a/.github/workflows/test_python_cplusplus.yml +++ b/.github/workflows/test_python_cplusplus.yml @@ -8,7 +8,7 @@ on: - master jobs: - ubuntu-cpp-tests: + build: name: C++ tests Ubuntu runs-on: ubuntu-22.04