diff --git a/.github/workflows/test_sbml_semantic_test_suite.yml b/.github/workflows/test_sbml_semantic_test_suite.yml index 3c0b3bd149..500ce07b7e 100644 --- a/.github/workflows/test_sbml_semantic_test_suite.yml +++ b/.github/workflows/test_sbml_semantic_test_suite.yml @@ -5,6 +5,7 @@ on: - develop - master - release** + - check_grad_sbml_test_suite pull_request: paths: - .github/workflows/test_sbml_semantic_test_suite.yml @@ -45,6 +46,8 @@ jobs: uses: ./.github/actions/install-apt-dependencies - run: AMICI_PARALLEL_COMPILE="" ./scripts/installAmiciSource.sh + - run: | + source build/venv/bin/activate && pip install git+https://github.com/ICB-DCM/fiddy.git - run: AMICI_PARALLEL_COMPILE="" ./scripts/run-SBMLTestsuite.sh ${{ matrix.cases }} - name: "Upload artifact: SBML semantic test suite results" diff --git a/python/sdist/setup.cfg b/python/sdist/setup.cfg index 0d27a0918e..cca53510db 100644 --- a/python/sdist/setup.cfg +++ b/python/sdist/setup.cfg @@ -55,6 +55,7 @@ test = pytest pytest-cov pytest-rerunfailures + pytest-xdist coverage shyaml antimony>=2.13 diff --git a/scripts/installAmiciSource.sh b/scripts/installAmiciSource.sh index bbb4bf4a83..caf7e062cf 100755 --- a/scripts/installAmiciSource.sh +++ b/scripts/installAmiciSource.sh @@ -30,7 +30,7 @@ else fi python -m pip install --upgrade pip wheel -python -m pip install --upgrade pip setuptools cmake_build_extension numpy +python -m pip install --upgrade pip setuptools cmake_build_extension numpy git+https://github.com/ICB-DCM/fiddy.git python -m pip install git+https://github.com/FFroehlich/pysb@fix_pattern_matching # pin to PR for SPM with compartments AMICI_BUILD_TEMP="${AMICI_PATH}/python/sdist/build/temp" \ python -m pip install --verbose -e "${AMICI_PATH}/python/sdist[petab,test,vis]" --no-build-isolation diff --git a/tests/conftest.py b/tests/conftest.py index 9b7dd7fb08..144fe5bae8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -70,6 +70,13 @@ def pytest_generate_tests(metafunc): # Get CLI option cases = metafunc.config.getoption("cases") if cases: + # iff specific case IDs are given and the SBML semantic test suite is not there, we should fail. + if not SBML_SEMANTIC_CASES_DIR.exists(): + raise ValueError( + "The SBML semantic cases are missing. You can install them with " + "'AMICI/scripts/run-SBMLTestsuite.sh'." + ) + # Run selected tests last_id = int(list(get_all_semantic_case_ids())[-1]) test_numbers = sorted(set(parse_selection(cases, last_id))) diff --git a/tests/testSBMLSuite.py b/tests/testSBMLSuite.py index 2feb3ea7e8..8fd43d0ae3 100755 --- a/tests/testSBMLSuite.py +++ b/tests/testSBMLSuite.py @@ -22,7 +22,9 @@ import pandas as pd import pytest from amici.constants import SymbolId -from amici.gradient_check import check_derivatives +from fiddy import MethodId, get_derivative +from fiddy.extensions.amici import reshape, run_amici_simulation_to_cached_functions +from fiddy.success import Consistency from numpy.testing import assert_allclose @@ -73,13 +75,29 @@ def test_sbml_testsuite_case( inplace=True, ) + # TODO remove after https://github.com/AMICI-dev/AMICI/pull/2101 + # and https://github.com/AMICI-dev/AMICI/issues/2106 + # Don't attempt to generate sensitivity code for models with events+algebraic rules, which will fail + sbml_file = find_model_file(current_test_path, test_id) + sbml_document = sbml.SBMLReader().readSBMLFromFile(str(sbml_file)) + sbml_model = sbml_document.getModel() + has_events = sbml_model.getNumEvents() > 0 + has_algebraic_rules = any( + rule.getTypeCode() == sbml.SBML_ALGEBRAIC_RULE + for rule in sbml_model.getListOfRules() + ) + generate_sensitivity_code = not (has_events and has_algebraic_rules) + # TODO https://github.com/AMICI-dev/AMICI/issues/2109 + generate_sensitivity_code &= test_id not in {"01240"} + # ^^^^^^^^ + # setup model model_dir = Path(__file__).parent / "SBMLTestModels" / test_id model, solver, wrapper = compile_model( current_test_path, test_id, model_dir, - generate_sensitivity_code=test_id in sensitivity_check_cases, + generate_sensitivity_code=generate_sensitivity_code, ) settings = read_settings_file(current_test_path, test_id) @@ -93,7 +111,7 @@ def test_sbml_testsuite_case( else: raise RuntimeError("Simulation failed unexpectedly") - # verify + # verify simulation results simulated = verify_results( settings, rdata, results, wrapper, model, atol, rtol ) @@ -101,11 +119,66 @@ def test_sbml_testsuite_case( # record results write_result_file(simulated, test_id, result_path) - # check sensitivities for selected models - if epsilon := sensitivity_check_cases.get(test_id): - solver.setSensitivityOrder(amici.SensitivityOrder.first) - solver.setSensitivityMethod(amici.SensitivityMethod.forward) - check_derivatives(model, solver, epsilon=epsilon) + # test sensitivities + if not model.getParameters(): + pytest.skip("No parameters -> no sensitivities to check") + + # TODO see https://github.com/AMICI-dev/AMICI/pull/2101 + if not generate_sensitivity_code: + pytest.skip("Sensitivity analysis is known to fail.") + if any(id_ == 0 for id_ in model.idlist): + pytest.skip("Sensitivity analysis for DAE is known to fail.") + + solver.setSensitivityOrder(amici.SensitivityOrder.first) + solver.setSensitivityMethod(amici.SensitivityMethod.forward) + # currently only checking "x"/"sx" for FSA + ( + amici_function_f, + amici_derivative_f, + structures_f, + ) = run_amici_simulation_to_cached_functions( + amici_model=model, + amici_solver=solver, + derivative_variables=["x"], + cache=False, + ) + rdata_f = amici.runAmiciSimulation(model, solver) + + # solver.setSensitivityMethod(amici.SensitivityMethod.adjoint) + # ( + # amici_function_a, + # amici_derivative_a, + # structures_a, + # ) = run_amici_simulation_to_cached_functions( + # amici_model=model, + # amici_solver=solver, + # derivative_variables=["x"], + # cache=False, + # ) + # rdata_a = amici.runAmiciSimulation(model, solver) + + point = np.asarray(model.getParameters()) + + derivative = get_derivative( + # can use `_f` or `_a` here, should be no difference + function=amici_function_f, + point=point, + sizes=[1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1], + direction_ids=model.getParameterIds(), + method_ids=[MethodId.FORWARD, MethodId.BACKWARD, MethodId.CENTRAL], + relative_sizes=True, + success_checker=Consistency(rtol=1e-2, atol=1e-4), + ) + + derivative_fd = reshape( + derivative.value.flat, structures_f["derivative"], sensitivities=True + )["x"] + derivative_fsa = rdata_f.sx + # derivative_asa = rdata_a.sllh # currently None, define some objective? + + # could alternatively use a `fiddy.DerivativeCheck` class + if not np.isclose(derivative_fd, derivative_fsa, rtol=5e-2, atol=5e-2).all(): + raise ValueError("Gradients were not validated.") except amici.sbml_import.SBMLException as err: pytest.skip(str(err))