Clean tests and divide them by environment for CI

.github/workflows/python-tox-testing-including-conda.yml → .github/workflows/python-tox-testing-base.yml

@@ -1,4 +1,4 @@
-name: Tests on poli (dev, conda, python 3.9)
+name: poli base (dev, conda, python 3.9)
 
 on: [push]
 
@@ -24,6 +24,6 @@ jobs:
     - name: Test linting with tox
       run: |
         tox -c tox.ini -e lint
-    - name: Test poli-base with tox (ignoring RaSP)
+    - name: Test basic behaviors of poli with tox and pytest
       run: |
-        tox -c tox.ini -e poli-base-py39 -- --ignore=src/poli/tests/registry/proteins/test_rasp.py
+        tox -c tox.ini -e poli-base-py39

.github/workflows/python-tox-testing-dockstring-env.yml

@@ -0,0 +1,26 @@
+name: poli dockstring (dev, conda, python 3.9)
+
+on: [push]
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 5
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.9'
+    - name: Add conda to system path
+      run: |
+        # $CONDA is an environment variable pointing to the root of the miniconda directory
+        echo $CONDA/bin >> $GITHUB_PATH
+    - name: Install dependencies
+      run: |
+        python -m pip install tox
+    - name: Test dockstring-related black boxes with tox and pytest
+      run: |
+        tox -c tox.ini -e poli-dockstring-py39

.github/workflows/python-tox-testing-lambo-env.yml

@@ -0,0 +1,26 @@
+name: poli lambo (dev, conda, python 3.9)
+
+on: [push]
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 5
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.9'
+    - name: Add conda to system path
+      run: |
+        # $CONDA is an environment variable pointing to the root of the miniconda directory
+        echo $CONDA/bin >> $GITHUB_PATH
+    - name: Install dependencies
+      run: |
+        python -m pip install tox
+    - name: Test lambo-related black boxes with tox and pytest
+      run: |
+        tox -c tox.ini -e poli-lambo-py39

.github/workflows/python-tox-testing-protein-env.yml

@@ -0,0 +1,26 @@
+name: poli protein (dev, conda, python 3.9)
+
+on: [push]
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 5
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.9'
+    - name: Add conda to system path
+      run: |
+        # $CONDA is an environment variable pointing to the root of the miniconda directory
+        echo $CONDA/bin >> $GITHUB_PATH
+    - name: Install dependencies
+      run: |
+        python -m pip install tox
+    - name: Test protein-related black boxes with tox and pytest
+      run: |
+        tox -c tox.ini -e poli-protein-py39

.github/workflows/python-tox-testing-tdc-env.yml

@@ -0,0 +1,26 @@
+name: poli tdc (dev, conda, python 3.9)
+
+on: [push]
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 5
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.9'
+    - name: Add conda to system path
+      run: |
+        # $CONDA is an environment variable pointing to the root of the miniconda directory
+        echo $CONDA/bin >> $GITHUB_PATH
+    - name: Install dependencies
+      run: |
+        python -m pip install tox
+    - name: Test PyTDC-related black boxes with tox and pytest
+      run: |
+        tox -c tox.ini -e poli-tdc-py39

CONTRIBUTING.md

@@ -3,48 +3,65 @@
 ![Linting: black](https://img.shields.io/badge/Linting-black-black)
 ![Testing: pytest](https://img.shields.io/badge/Testing-pytest-blue)
 ![Testing: tox](https://img.shields.io/badge/Testing-tox-blue)
-![Main branch: black](https://img.shields.io/badge/Pull_request_to-dev-blue)
+![Main branch: dev](https://img.shields.io/badge/Pull_request_to-dev-blue)
 
 This note details how to contribute to `poli`.
 
 ## Forking and making pull requests
 
 The main development branch is called `dev`. To contribute, we recommend creating a fork of this repository and making changes on your version. Once you are ready to contribute, we expect you to document, lint and test.
 
-## Documentation standards
+## Installing dev dependencies and pre-commit hooks
 
-We follow [numpy's documentation standards](https://numpydoc.readthedocs.io/en/latest/format.html).
+We recommend you create a `poli-dev` environment in conda
 
-## Linting your changes
+```bash
+conda create -n poli-dev python=3.10
+conda activate poli-dev
+pip install -r requirements.txt
+pip install -r requirements-dev.txt
+```
 
-We expect you to lint the code you write or modify using `black`.
+The dev requirements include `pre-commit`. Install the hooks in our config by running
 
 ```bash
-pip install black
-black ./path/to/files
+pre-commit install
 ```
 
-## Testing your changes for `dev``
+Now every commit will run linting and isorting for you. You can also run it manually by saying
+
+```bash
+pre-commit run --all-files
+```
+
+## Documentation standards
+
+We follow [numpy's documentation standards](https://numpydoc.readthedocs.io/en/latest/format.html).
+
+## Creating a new environment? Mark your tests
+
+If you're contributing a black box in a new environment, remember to
+
+1. Mark all your tests with `@pyest.mark.poli__your_env`.
+2. Add a description of your marker to the `[tool.pytest.ini_options]`.
+
+## Testing your changes for `dev`
 
 Since we are testing multiple conda environments, we settled for using a combination of `tox` and `pytest`.
 
 ```bash
 pip install tox
 
 # To test linting (from the root of the project)
-tox -c tox.dev.ini -e lint
+tox -c tox.ini -e lint
 
 # To test in the base environment for poli
-tox -c tox.dev.ini -e poli-base-py39
+tox -c tox.ini -e poli-base-py39
 ```
 
-If you want to run tests in all environments, remove `-e poli-base-py39` and just run `tox`.
-
-## More thorough testing
-
-In many cases, testing with the instructions above should be enough. However, since we are dealing with creating conda environments, the definite test comes by building the Docker image specified in `Dockerfile.test`, and running it.
+There are several different environments (depending on the black boxes we test). Check `tox.ini` for more details.
 
-When contributing to the `@master` branch (i.e. to release), we will run these tests.
+If you want to run tests in all environments, remove `-e poli-base-py39` and just run `tox`. This might take a while, and several conda envs will be created.
 
 ## Create a pull request to dev
 

pyproject.toml

@@ -37,6 +37,12 @@ Homepage = "https://github.com/MachineLearningLifeScience/poli"
 [tool.pytest.ini_options]
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "poli__lambo: marks tests that run in the poli__lambo environment",
+    "poli__dockstring: marks tests that run in the poli__dockstring environment",
+    "poli__tdc: marks tests that run in the poli__tdc environment",
+    "poli__protein: marks tests that run in the poli__protein environment",
+    "poli__rasp: marks tests that run in the poli__rasp environment",
+    "unmarked: All other tests, which usually run in the base environment",
 ]
 
 [tool.isort]

src/poli/core/black_box_information.py

@@ -14,6 +14,8 @@
 
 from typing import Literal, Union
 
+import numpy as np
+
 
 class BlackBoxInformation:
     def __init__(
@@ -145,7 +147,11 @@ def __repr__(self):
     def as_dict(self):
         return {
             "name": self.name,
-            "max_sequence_length": self.max_sequence_length,
+            "max_sequence_length": (
+                self.max_sequence_length
+                if self.max_sequence_length != np.inf
+                else "inf"
+            ),
             "aligned": self.aligned,
             "fixed_length": self.fixed_length,
             "deterministic": self.deterministic,

src/poli/tests/benchmarks/test_benchmark_creation.py

@@ -7,6 +7,8 @@
 tests are already covered by the tests in objective_repository.
 """
 
+import pytest
+
 
 def test_creating_toy_continuous_functions_benchmark():
     from poli.benchmarks import ToyContinuousFunctionsBenchmark
@@ -27,6 +29,7 @@ def test_creating_embedded_toy_continuous_functions_benchmark():
             f(x0)
 
 
+@pytest.mark.poli__tdc
 def test_creating_guacamol_benchmark():
     from poli.benchmarks import GuacaMolGoalDirectedBenchmark
 
@@ -35,7 +38,12 @@ def test_creating_guacamol_benchmark():
     for problem in benchmark:
         f, x0 = problem.black_box, problem.x0
 
+        # Break after the first iteration
+        # for CI efficiency
+        break
+
 
+@pytest.mark.poli__tdc
 def test_creating_pmo_benchmark():
     from poli.benchmarks import PMOBenchmark
 
@@ -44,6 +52,8 @@ def test_creating_pmo_benchmark():
     for problem in benchmark:
         f, x0 = problem.black_box, problem.x0
 
-
-if __name__ == "__main__":
-    test_creating_embedded_toy_continuous_functions_benchmark()
+        # Break after the first iteration
+        # for CI efficiency. The creation of all
+        # these black boxes is already being tested
+        # in the chemistry test suite.
+        break

src/poli/tests/conftest.py

@@ -0,0 +1,11 @@
+def pytest_collection_modifyitems(items, config):
+    for item in items:
+        markers = item.iter_markers()
+        is_unmarked = True
+        for marker in markers:
+            if marker.name != "parametrize":
+                is_unmarked = False
+                break
+
+        if is_unmarked:
+            item.add_marker("unmarked")