Fix venv related installation issues (#2356)

* fix venv and BLAS

* fixup

* fixup?

* Update runNotebook.sh

* revert BLAS

* remove virtualenv

* remove venv creation from installAmiciSource

* clear venv in buildInstall.sh

* cleanup

* doh

* add venv back to installAmiciSource

* readd numpy

* export python executable in source install

.github/workflows/test_petab_test_suite.yml

@@ -57,27 +57,27 @@ jobs:
 
       - name: Install petab
         run: |
-          source ./build/venv/bin/activate \
+          source ./venv/bin/activate \
             && pip3 install wheel pytest shyaml pytest-cov pysb
 
       # retrieve test models
       - name: Download and install PEtab test suite
         run: |
           git clone --depth 1 --branch main \
             https://github.com/PEtab-dev/petab_test_suite \
-            && source ./build/venv/bin/activate \
+            && source ./venv/bin/activate \
             && cd petab_test_suite && pip3 install -e .
 
       - name: Run PEtab-related unit tests
         run: |
-          source ./build/venv/bin/activate \
+          source ./venv/bin/activate \
             && pytest --cov-report=xml:coverage.xml \
               --cov=./ python/tests/test_*petab*.py python/tests/petab/
 
       # run test models
       - name: Run PEtab test suite
         run: |
-          source ./build/venv/bin/activate \
+          source ./venv/bin/activate \
           && AMICI_PARALLEL_COMPILE="" pytest -v \
             --cov-report=xml:coverage.xml \
             --cov-append \

.github/workflows/test_python_cplusplus.yml

@@ -45,11 +45,11 @@ jobs:
       run: scripts/installAmiciSource.sh
 
     - name: Check OpenMP support
-      run: source build/venv/bin/activate && python -c "import amici; import sys; sys.exit(not amici.compiledWithOpenMP())"
+      run: source venv/bin/activate && python -c "import amici; import sys; sys.exit(not amici.compiledWithOpenMP())"
 
     - name: Python tests (part 1)
       run: |
-        source build/venv/bin/activate \
+        source venv/bin/activate \
         && pytest \
           --ignore-glob=*petab* \
           --ignore-glob=*test_splines.py \
@@ -64,7 +64,7 @@ jobs:
     - name: Python tests splines
       if: ${{ github.base_ref == 'master' || github.event.merge_group.base_ref == 'master'}}
       run: |
-        source build/venv/bin/activate \
+        source venv/bin/activate \
         && pytest \
           --cov=amici \
           --cov-report=xml:"${AMICI_DIR}/build/coverage_py.xml" \
@@ -138,7 +138,7 @@ jobs:
 
     - name: Python tests
       run: |
-        source build/venv/bin/activate \
+        source venv/bin/activate \
         && pytest \
           --cov=amici \
           --cov-report=xml:"${AMICI_DIR}/build/coverage_py.xml" \
@@ -214,7 +214,7 @@ jobs:
 
     - name: Install notebook dependencies
       run: |
-        source build/venv/bin/activate \
+        source venv/bin/activate \
         && pip install jax[cpu]
 
     - name: example notebooks
@@ -251,7 +251,7 @@ jobs:
       run: scripts/installAmiciSource.sh
 
     - name: Check OpenMP support
-      run: source build/venv/bin/activate && python -c "import amici; import sys; sys.exit(not amici.compiledWithOpenMP())"
+      run: source venv/bin/activate && python -c "import amici; import sys; sys.exit(not amici.compiledWithOpenMP())"
 
     - name: cppcheck
       run: scripts/run-cppcheck.sh
@@ -296,7 +296,7 @@ jobs:
         scripts/installAmiciSource.sh
 
     - name: Check OpenMP support
-      run: source build/venv/bin/activate && python -c "import amici; import sys; sys.exit(not amici.compiledWithOpenMP())"
+      run: source venv/bin/activate && python -c "import amici; import sys; sys.exit(not amici.compiledWithOpenMP())"
 
     - name: Get BioNetGen
       run: scripts/buildBNGL.sh

.github/workflows/test_python_ver_matrix.yml

@@ -51,15 +51,15 @@ jobs:
       run: scripts/installAmiciSource.sh
 
     # install pysb before sympy to allow for sympy>=1.12 (https://github.com/pysb/pysb/commit/e83937cb8c74afc9b2fa96595b68464946745f33)
-    - run: source build/venv/bin/activate && pip3 install git+https://github.com/pysb/pysb
+    - run: source venv/bin/activate && pip3 install git+https://github.com/pysb/pysb
 
     # until sympy>1.12 is released
-    - run: source build/venv/bin/activate && pip3 install git+https://github.com/sympy/sympy.git@master
+    - run: source venv/bin/activate && pip3 install git+https://github.com/sympy/sympy.git@master
       if: matrix.python-version == '3.12'
 
     - name: Python tests
       run: |
-        source build/venv/bin/activate \
+        source venv/bin/activate \
           && python3 -m pytest \
             --durations=10 \
             --ignore-glob=*petab* \

.gitignore

@@ -202,3 +202,4 @@ CS_Signalling_ERBB_RAS_AKT/
 cache_fiddy/*
 debug/*
 tests/benchmark-models/cache_fiddy/*
+venv/*

scripts/buildAmici.sh

@@ -25,7 +25,29 @@ else
 fi
 
 # required for build swig interface
-pip show numpy > /dev/null || python3 -m pip install numpy
+venv_dir="${amici_path}/venv"
+set +e
+mkdir -p "${venv_dir}"
+python3 -m venv "${venv_dir}" --clear
+# in case this fails (usually due to missing ensurepip, try getting pip
+# manually
+if [[ $? ]]; then
+    set -e
+    python3 -m venv "${venv_dir}" --clear --without-pip
+    source "${venv_dir}/bin/activate"
+    get_pip=${amici_path}/get-pip.py
+    curl "https://bootstrap.pypa.io/get-pip.py" -o "${get_pip}"
+    python3 "${get_pip}"
+    rm "${get_pip}"
+else
+    set -e
+    source "${venv_dir}/bin/activate"
+fi
+
+# set python executable for cmake
+export PYTHON_EXECUTABLE="${amici_path}/venv/bin/python"
+# install numpy
+python3 -m pip install numpy
 
 ${cmake} \
   -Wdev -DAMICI_CXX_OPTIONS="-Wall;-Wextra${extra_cxx_flags}" \

scripts/installAmiciArchive.sh

@@ -18,18 +18,18 @@ rm -f ${AMICI_PATH}/python/sdist/amici/amici_without_hdf5.py
 
 # test install from archive
 set +e
-python3 -m venv ${AMICI_PATH}/build/venvArchive --clear
+python3 -m venv ${AMICI_PATH}/venvArchive --clear
 # in case this fails (usually due to missing ensurepip, try getting pip
 # manually
 if [[ $? ]]; then
     set -e
-    python3 -m venv ${AMICI_PATH}/build/venvArchive --clear --without-pip
-    source ${AMICI_PATH}/build/venvArchive/bin/activate
+    python3 -m venv ${AMICI_PATH}/venvArchive --clear --without-pip
+    source ${AMICI_PATH}/venvArchive/bin/activate
     curl https://bootstrap.pypa.io/get-pip.py -o ${AMICI_PATH}/build/get-pip.py
     python ${AMICI_PATH}/build/get-pip.py
 else
     set -e
-    source ${AMICI_PATH}/build/venvArchive/bin/activate
+    source ${AMICI_PATH}/venvArchive/bin/activate
 fi
 
 pip install $(ls -t ${AMICI_PATH}/build/python/amici-*.tar.gz | head -1)

scripts/installAmiciSource.sh

@@ -5,12 +5,12 @@ set -e
 SCRIPT_PATH=$(dirname $BASH_SOURCE)
 AMICI_PATH=$(cd "$SCRIPT_PATH/.." && pwd)
 
-venv_dir="${AMICI_PATH}/build/venv"
 # Disabled until cmake package is made compatible with updated setup.py
 #make python-wheel
 #pip3 install --user --prefix= `ls -t ${AMICI_PATH}/build/python/amici-*.whl | head -1`
 
 # test install from setup.py
+venv_dir="${AMICI_PATH}/venv"
 set +e
 mkdir -p "${venv_dir}"
 python3 -m venv "${venv_dir}" --clear
@@ -20,7 +20,7 @@ if [[ $? ]]; then
     set -e
     python3 -m venv "${venv_dir}" --clear --without-pip
     source "${venv_dir}/bin/activate"
-    get_pip=${AMICI_PATH}/build/get-pip.py
+    get_pip=${AMICI_PATH}/get-pip.py
     curl "https://bootstrap.pypa.io/get-pip.py" -o "${get_pip}"
     python3 "${get_pip}"
     rm "${get_pip}"
@@ -29,6 +29,9 @@ else
     source "${venv_dir}/bin/activate"
 fi
 
+# set python executable for cmake
+export PYTHON_EXECUTABLE="${AMICI_PATH}/venv/bin/python"
+
 python -m pip install --upgrade pip wheel
 python -m pip install --upgrade pip setuptools cmake_build_extension numpy
 python -m pip install git+https://github.com/FFroehlich/pysb@fix_pattern_matching # pin to PR for SPM with compartments

scripts/run-SBMLTestsuite.sh

@@ -9,7 +9,7 @@ if [[ ! -d "tests/sbml-test-suite" ]]; then
     mv -f ./sbml-test-suite ./tests/sbml-test-suite
 fi
 
-source build/venv/bin/activate
+source venv/bin/activate
 pip show pytest-xdist > /dev/null 2>&1 || pip install pytest-xdist
 pip install coverage pytest-cov
 

scripts/run-codecov.sh

@@ -4,7 +4,7 @@
 script_path=$(dirname $BASH_SOURCE)
 amici_path=$(cd "$script_path"/.. && pwd)
 
-source "${amici_path}"/build/venv/bin/activate
+source "${amici_path}"/venv/bin/activate
 pip install coverage pytest pytest-cov
 
 if [[ -z "${BNGPATH}" ]]; then

scripts/run-python-tests.sh

@@ -12,7 +12,7 @@ if [[ -z "${BNGPATH}" ]]; then
 fi
 
 cd "${amici_path}"/python/tests
-source "${amici_path}"/build/venv/bin/activate
+source "${amici_path}"/venv/bin/activate
 
 # PEtab tests are run separately
 pytest \

scripts/run-valgrind-py.sh

@@ -15,7 +15,7 @@ if [ $# -eq 0 ]
   then
     # No arguments supplied, run all tests
     cd "${amici_path}"/python/tests
-    source "${amici_path}"/build/venv/bin/activate
+    source "${amici_path}"/venv/bin/activate
     command=(python -m pytest -vv --ignore-glob=*petab* -W 'ignore:Signature ')
     #                                                    ^ ignores the following warning that occurs only under valgrind,
     # e.g. `valgrind python -c "import h5py"`:

scripts/runNotebook.sh

@@ -25,7 +25,7 @@ if [ $# -eq 0 ]; then
     exit 1
 fi
 
-source ${AMICI_PATH}/build/venv/bin/activate
+source ${AMICI_PATH}/venv/bin/activate
 pip3 show nbconvert || pip3 install --upgrade nbconvert
 pip3 show ipykernel || (pip3 install --upgrade ipykernel && python3 -m ipykernel install --user --name amici --display-name "Python (amici)")