From a07cdac3a35eb57b37d7991c5229e8efa16f1d37 Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Sat, 8 Jun 2024 20:56:11 +0200 Subject: [PATCH 1/6] update CI a bit --- .github/workflows/ci.yml | 23 +-- .gitlab-ci.yml | 307 --------------------------------------- 2 files changed, 8 insertions(+), 322 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 703836343..3f2fb612f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,10 @@ name: CI + on: push: branches: - master - - develop + pull_request: jobs: test: @@ -13,8 +14,8 @@ jobs: fail-fast: false matrix: version: - - '1.6' # Oldest supported version for COBREXA.jl - '1' # This is always the latest stable release in the 1.X series + - '1.6' # LTS #- 'nightly' os: - ubuntu-latest @@ -23,21 +24,12 @@ jobs: arch: - x64 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: actions/cache@v1 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- + - uses: julia-actions/cache@v1 - uses: julia-actions/julia-buildpkg@latest - run: | git config --global user.name Tester @@ -45,6 +37,7 @@ jobs: - uses: julia-actions/julia-runtest@latest continue-on-error: ${{ matrix.version == 'nightly' }} - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v4 with: - file: lcov.info + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 4971d5432..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,307 +0,0 @@ - -stages: - - test # this checks the viability of the code - - assets # this builds assets to be included in documentation and distribution binaries - - documentation # this processes the documentation - - test-compat # this runs many additional compatibility tests - -variables: - GIT_STRATEGY: clone - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - APPTAINER_DOCKER_TAG: "v3.9.4" - DOCKER_HUB_TAG: "lcsbbiocore/cobrexa.jl" - DOCKER_GHCR_TAG: "ghcr.io/lcsb-biocore/docker/cobrexa.jl" - APPTAINER_GHCR_TAG: "lcsb-biocore/apptainer/cobrexa.jl" - -# -# Predefined conditions for triggering jobs -# - -.global_trigger_pull_request: &global_trigger_pull_request - rules: - - if: $CI_COMMIT_BRANCH == "develop" - when: never - - if: $CI_COMMIT_BRANCH == "master" - when: never - - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" - -.global_trigger_build_doc: &global_trigger_build_doc - rules: - - if: $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "master" && $CI_EXTERNAL_PULL_REQUEST_SOURCE_BRANCH_NAME == "develop" - - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" - when: never - - if: $CI_COMMIT_BRANCH == "develop" - - if: $CI_COMMIT_BRANCH == "master" - - if: $CI_COMMIT_TAG =~ /^v/ - -.global_trigger_full_tests: &global_trigger_full_tests - rules: - - if: $CI_COMMIT_BRANCH == "develop" - - if: $CI_COMMIT_BRANCH == "master" - - if: $CI_COMMIT_TAG =~ /^v/ - -.global_trigger_compat_tests: &global_trigger_compat_tests - rules: - - if: $CI_COMMIT_BRANCH == "master" - - if: $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "master" - -.global_trigger_test_containers: &global_trigger_test_containers - rules: - - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" - when: never - - if: $CI_COMMIT_BRANCH == "develop" - -.global_trigger_release_containers: &global_trigger_release_containers - rules: - - if: $CI_COMMIT_TAG =~ /^v/ - -# -# Test environment & platform settings -# - -.global_dind: &global_dind - image: docker:20.10.12 - tags: - - privileged - services: - - name: docker:20.10.12-dind - command: ["--tls=false", "--mtu=1458", "--registry-mirror", "https://docker-registry.lcsb.uni.lu"] - before_script: - - docker login -u $CI_USER_NAME -p $GITLAB_ACCESS_TOKEN $CI_REGISTRY - -.global_julia18: &global_julia18 - variables: - JULIA_VER: "v1.8.3" - -.global_julia16: &global_julia16 - variables: - JULIA_VER: "v1.6.0" - -.global_env_linux: &global_env_linux - script: - - $ARTENOLIS_SOFT_PATH/julia/$JULIA_VER/bin/julia --inline=yes --check-bounds=yes --color=yes --project=@. -e 'import Pkg; Pkg.test(; coverage = true)' - -.global_env_win: &global_env_win - script: - - $global:LASTEXITCODE = 0 # Note the global prefix. - - Invoke-Expression $Env:ARTENOLIS_SOFT_PATH"\julia\"$Env:JULIA_VER"\bin\julia --inline=yes --check-bounds=yes --color=yes --project=@. -e 'import Pkg; Pkg.test(; coverage = true)'" - - exit $LASTEXITCODE - -.global_env_win10: &global_env_win10 - tags: - - windows10 - <<: *global_env_win - -.global_env_mac: &global_env_mac - tags: - - mac - script: - - $ARTENOLIS_SOFT_PATH_MAC/julia/$JULIA_VER/Contents/Resources/julia/bin/julia --inline=yes --check-bounds=yes --color=yes --project=@. -e 'import Pkg; Pkg.test(; coverage = true)' - -.global_build_apptainer: &global_build_apptainer - image: - name: "quay.io/singularity/singularity:$APPTAINER_DOCKER_TAG" - # the image entrypoint is the singularity binary by default - entrypoint: ["/bin/sh", "-c"] - tags: - - privileged - -# -# TESTS -# -# The "basic" required test that gets triggered for the basic testing, runs in -# any available docker and current julia -# - -docker:julia1.8: - stage: test - image: $CI_REGISTRY/r3/docker/julia-custom - script: - - julia --check-bounds=yes --inline=yes --project=@. -e "import Pkg; Pkg.test(; coverage = true)" - after_script: - - julia --project=test/coverage test/coverage/coverage-summary.jl - <<: *global_trigger_pull_request - -# -# The required compatibility test to pass on branches&tags before the docs get -# built & deployed -# - -linux:julia1.8: - stage: test - tags: - - slave01 - <<: *global_trigger_full_tests - <<: *global_julia18 - <<: *global_env_linux - -linux:julia1.6: - stage: test - tags: - - slave02 - <<: *global_trigger_full_tests - <<: *global_julia16 - <<: *global_env_linux - -# -# Additional platform&environment compatibility tests -# - -windows10:julia1.8: - stage: test-compat - <<: *global_trigger_compat_tests - <<: *global_julia18 - <<: *global_env_win10 - -mac:julia1.8: - stage: test-compat - <<: *global_trigger_compat_tests - <<: *global_julia18 - <<: *global_env_mac - -windows10:julia1.6: - stage: test-compat - <<: *global_trigger_compat_tests - <<: *global_julia16 - <<: *global_env_win10 - -mac:julia1.6: - stage: test-compat - <<: *global_trigger_compat_tests - <<: *global_julia16 - <<: *global_env_mac - -# -# ASSETS -# -# This builds the development history gif using gource, and some containers. -# - -gource: - stage: assets - needs: [] # allow faster start - script: - - docker run -v "$PWD":/visualization $CI_REGISTRY/r3/docker/gource - artifacts: - paths: ['output.gif'] - expire_in: 1 year - <<: *global_trigger_build_doc - <<: *global_dind - -apptainer-test: - stage: assets - script: | - alias apptainer=singularity - apptainer build cobrexa-test.sif cobrexa.def - <<: *global_build_apptainer - <<: *global_trigger_test_containers - -apptainer-release: - stage: assets - script: - - | - # build the container - alias apptainer=singularity - apptainer build cobrexa.sif cobrexa.def - - | - # push to GHCR - alias apptainer=singularity - export SINGULARITY_DOCKER_USERNAME="$GITHUB_ACCESS_USERNAME" - export SINGULARITY_DOCKER_PASSWORD="$GITHUB_ACCESS_TOKEN" - apptainer push cobrexa.sif "oras://ghcr.io/$APPTAINER_GHCR_TAG:latest" - apptainer push cobrexa.sif "oras://ghcr.io/$APPTAINER_GHCR_TAG:$CI_COMMIT_TAG" - <<: *global_build_apptainer - <<: *global_trigger_release_containers - -docker-test: - stage: assets - script: - - docker build -t "$DOCKER_HUB_TAG:testing" . - <<: *global_dind - <<: *global_trigger_test_containers - -docker-release: - stage: assets - script: - - docker build -t "$DOCKER_HUB_TAG:latest" . - # alias and push to docker hub - - docker tag "$DOCKER_HUB_TAG:latest" "$DOCKER_HUB_TAG:$CI_COMMIT_TAG" - - echo "$DOCKER_IO_ACCESS_TOKEN" | docker login --username "$DOCKER_IO_USER" --password-stdin - - docker push "$DOCKER_HUB_TAG:latest" - - docker push "$DOCKER_HUB_TAG:$CI_COMMIT_TAG" - # make 2 extra aliases and push to GHCR - - docker tag "$DOCKER_HUB_TAG:latest" "$DOCKER_GHCR_TAG:latest" - - docker tag "$DOCKER_HUB_TAG:latest" "$DOCKER_GHCR_TAG:$CI_COMMIT_TAG" - - echo "$GITHUB_ACCESS_TOKEN" | docker login ghcr.io --username "$GITHUB_ACCESS_USERNAME" --password-stdin - - docker push "$DOCKER_GHCR_TAG:latest" - - docker push "$DOCKER_GHCR_TAG:$CI_COMMIT_TAG" - <<: *global_dind - <<: *global_trigger_release_containers - -# -# DOCUMENTATION TESTS -# -# In pull requests, triggered after the tests succeed to avoid unnecessary -# double failure. In normal branch testing, these get triggered with normal -# tests (the error should be visible ASAP). We avoid a separate stage to keep -# the pipeline parallelizable. -# - -.global_doctests: &global_doctests - image: $CI_REGISTRY/r3/docker/julia-custom - script: - - julia --project=@. -e 'import Pkg; Pkg.instantiate();' - - julia --project=@. --color=yes test/doctests.jl - -doc-tests-pr:julia1.8: - stage: documentation - <<: *global_doctests - <<: *global_trigger_pull_request - -doc-tests:julia1.8: - stage: test - <<: *global_doctests - <<: *global_trigger_full_tests - -# -# DOCUMENTATION -# - -pages: - stage: documentation - dependencies: - - gource - # Note: This dependency is also implied by the stage ordering, but let's - # be sure. As of Nov 2021, the assets are not used directly, but referred - # to externally from the docs. - image: $CI_REGISTRY/r3/docker/julia-custom - script: - # resolve and build packages from the docs/Project.toml file - - julia --project=docs -e 'using Pkg; Pkg.resolve(); Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate();' - - # build and deploy docs (this doesn't upload the gource animation asset) - - julia --project=docs --color=yes docs/make.jl - - # move to the directory to be picked up by Gitlab pages (with assets) - - mv docs/build public - artifacts: - paths: - - public - <<: *global_trigger_build_doc - -# -# EXTERNAL REPOSITORIES -# -# This trigger the test pipeline in external repo as defined by gitlab -# variables. -# - -trigger: - stage: test-compat - image: curlimages/curl - tags: - - privileged - script: - - curl --silent --output /dev/null -X POST -F token=$EXTERNAL_REPO_TOKEN -F ref=$EXTERNAL_REPO_BRANCH $EXTERNAL_REPO - <<: *global_trigger_full_tests From a6adea249098ce9b344b086da0fef76b7b302215 Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Sat, 18 May 2024 19:46:50 +0200 Subject: [PATCH 2/6] work around equality bounds in coupling (as produced by mgPipe) --- src/base/types/MATModel.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/base/types/MATModel.jl b/src/base/types/MATModel.jl index 958d9d3be..55cd0411f 100644 --- a/src/base/types/MATModel.jl +++ b/src/base/types/MATModel.jl @@ -108,6 +108,11 @@ function coupling_bounds(m::MATModel) sparse(fill(-Inf, nc)), sparse(reshape(m.mat["b"], length(m.mat["b"]))[n_reactions(m)+1:end]), ) + elseif haskey(m.mat, "d") + ( + sparse(reshape(get(m.mat, "d", fill(-Inf, nc, 1)), nc)), + sparse(reshape(get(m.mat, "d", fill(Inf, nc, 1)), nc)), + ) else ( sparse(reshape(get(m.mat, "cl", fill(-Inf, nc, 1)), nc)), From 93a4e37431c04a138462700891e3cdcbd4d645d5 Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Sat, 18 May 2024 19:59:24 +0200 Subject: [PATCH 3/6] this did not sound right at all --- src/base/types/MATModel.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/base/types/MATModel.jl b/src/base/types/MATModel.jl index 55cd0411f..05a557a05 100644 --- a/src/base/types/MATModel.jl +++ b/src/base/types/MATModel.jl @@ -105,8 +105,8 @@ function coupling_bounds(m::MATModel) nc = n_coupling_constraints(m) if _mat_has_squashed_coupling(m.mat) ( - sparse(fill(-Inf, nc)), - sparse(reshape(m.mat["b"], length(m.mat["b"]))[n_reactions(m)+1:end]), + reshape(m.mat["b"], length(m.mat["b"]))[n_reactions(m)+1:end], + reshape(m.mat["b"], length(m.mat["b"]))[n_reactions(m)+1:end], ) elseif haskey(m.mat, "d") ( From d122cea863ebbdb3f7af8723aebdb7deabe56394 Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Sat, 18 May 2024 19:59:47 +0200 Subject: [PATCH 4/6] desparsify coupling bounds --- src/base/types/MATModel.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/base/types/MATModel.jl b/src/base/types/MATModel.jl index 05a557a05..ebdf968ff 100644 --- a/src/base/types/MATModel.jl +++ b/src/base/types/MATModel.jl @@ -110,13 +110,13 @@ function coupling_bounds(m::MATModel) ) elseif haskey(m.mat, "d") ( - sparse(reshape(get(m.mat, "d", fill(-Inf, nc, 1)), nc)), - sparse(reshape(get(m.mat, "d", fill(Inf, nc, 1)), nc)), + reshape(get(m.mat, "d", fill(-Inf, nc, 1)), nc), + reshape(get(m.mat, "d", fill(Inf, nc, 1)), nc), ) else ( - sparse(reshape(get(m.mat, "cl", fill(-Inf, nc, 1)), nc)), - sparse(reshape(get(m.mat, "cu", fill(Inf, nc, 1)), nc)), + reshape(get(m.mat, "cl", fill(-Inf, nc, 1)), nc), + reshape(get(m.mat, "cu", fill(Inf, nc, 1)), nc), ) end end From 0a9268a9fb78a687ce1f1428805b0efe5d600b1a Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Sat, 18 May 2024 20:20:45 +0200 Subject: [PATCH 5/6] ok here I discovered `dsense` --- src/base/types/MATModel.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/base/types/MATModel.jl b/src/base/types/MATModel.jl index ebdf968ff..b9f9c059e 100644 --- a/src/base/types/MATModel.jl +++ b/src/base/types/MATModel.jl @@ -104,14 +104,18 @@ Extracts the coupling constraints. Currently, there are several accepted ways to function coupling_bounds(m::MATModel) nc = n_coupling_constraints(m) if _mat_has_squashed_coupling(m.mat) + c = reshape(m.mat["b"], length(m.mat["b"]))[n_reactions(m)+1:end] + csense = reshape(m.mat["csense"], length(m.mat["csense"]))[n_reactions(m)+1:end], ( - reshape(m.mat["b"], length(m.mat["b"]))[n_reactions(m)+1:end], - reshape(m.mat["b"], length(m.mat["b"]))[n_reactions(m)+1:end], + [sense in ["G", "E"] ? val : -Inf for (val, sense) in zip(c, csense)], + [sense in ["L", "E"] ? val : Inf for (val, sense) in zip(c, csense)], ) - elseif haskey(m.mat, "d") + elseif haskey(m.mat, "d") && haskey(m.mat, "dsense") + d = reshape(m.mat["d"], nc) + dsense = reshape(m.mat["dsense"], nc) ( - reshape(get(m.mat, "d", fill(-Inf, nc, 1)), nc), - reshape(get(m.mat, "d", fill(Inf, nc, 1)), nc), + [sense in ["G", "E"] ? val : -Inf for (val, sense) in zip(d, dsense)], + [sense in ["L", "E"] ? val : Inf for (val, sense) in zip(d, dsense)], ) else ( From 9db803f69d542ff29e66ef02d1959a25da1e572a Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Sat, 18 May 2024 20:28:48 +0200 Subject: [PATCH 6/6] deal with infinities --- src/base/solver.jl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/base/solver.jl b/src/base/solver.jl index 7d112e7d8..bad57c10c 100644 --- a/src/base/solver.jl +++ b/src/base/solver.jl @@ -24,8 +24,16 @@ function make_optimization_model(model::MetabolicModel, optimizer; sense = MAX_S C = coupling(model) # empty if no coupling isempty(C) || begin cl, cu = coupling_bounds(model) - @constraint(optimization_model, c_lbs, cl .<= C * x) # coupling lower bounds - @constraint(optimization_model, c_ubs, C * x .<= cu) # coupling upper bounds + @constraint( + optimization_model, + c_lbs, + cl[isfinite.(cl)] .<= C[isfinite.(cl), :] * x + ) # coupling lower bounds + @constraint( + optimization_model, + c_ubs, + C[isfinite.(cu), :] * x .<= cu[isfinite.(cu)] + ) # coupling upper bounds end return optimization_model