From 72863402923dc533f0b70b09399fe050dc338aa3 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Wed, 20 Sep 2023 10:47:11 -0700 Subject: [PATCH 1/4] add a --disable-backend option that adds backends to be disabled --- tests/conftest.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 673d30d3b6..1338e7d36a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,17 @@ import pyhf +def pytest_addoption(parser): + parser.addoption( + "--disable-backend", + action="append", + type=str, + default=[], + choices=["tensorflow", "pytorch", "jax", "minuit"], + help="list of backends to disable in tests", + ) + + # Factory as fixture pattern @pytest.fixture def get_json_from_tarfile(): @@ -59,14 +70,14 @@ def reset_backend(): @pytest.fixture( scope='function', params=[ - (pyhf.tensor.numpy_backend(), None), - (pyhf.tensor.pytorch_backend(), None), - (pyhf.tensor.pytorch_backend(precision='64b'), None), - (pyhf.tensor.tensorflow_backend(), None), - (pyhf.tensor.jax_backend(), None), + (("numpy_backend", dict()), ("scipy_optimizer", dict())), + (("pytorch_backend", dict()), ("scipy_optimizer", dict())), + (("pytorch_backend", dict(precision="64b")), ("scipy_optimizer", dict())), + (("tensorflow_backend", dict()), ("scipy_optimizer", dict())), + (("jax_backend", dict()), ("scipy_optimizer", dict())), ( - pyhf.tensor.numpy_backend(poisson_from_normal=True), - pyhf.optimize.minuit_optimizer(), + ("numpy_backend", dict(poisson_from_normal=True)), + ("minuit_optimizer", dict()), ), ], ids=['numpy', 'pytorch', 'pytorch64', 'tensorflow', 'jax', 'numpy_minuit'], @@ -87,13 +98,20 @@ def backend(request): only_backends = [ pid for pid in param_ids if request.node.get_closest_marker(f'only_{pid}') ] + disable_backend = any( + backend in param_id for backend in request.config.disable_backend + ) if skip_backend and (param_id in only_backends): raise ValueError( f"Must specify skip_{param_id} or only_{param_id} but not both!" ) - if skip_backend: + if disable_backend: + pytest.skip( + f"skipping {func_name} as the backend is disabled: {request.config.disable_backend}" + ) + elif skip_backend: pytest.skip(f"skipping {func_name} as specified") elif only_backends and param_id not in only_backends: pytest.skip( From 360282dabe0dc923a948200c7334afae987dfb9d Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Wed, 20 Sep 2023 10:47:52 -0700 Subject: [PATCH 2/4] typo --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1338e7d36a..1771b98150 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -99,7 +99,7 @@ def backend(request): pid for pid in param_ids if request.node.get_closest_marker(f'only_{pid}') ] disable_backend = any( - backend in param_id for backend in request.config.disable_backend + backend in param_id for backend in request.config.option.disable_backend ) if skip_backend and (param_id in only_backends): @@ -109,7 +109,7 @@ def backend(request): if disable_backend: pytest.skip( - f"skipping {func_name} as the backend is disabled: {request.config.disable_backend}" + f"skipping {func_name} as the backend is disabled: {request.config.option.disable_backend}" ) elif skip_backend: pytest.skip(f"skipping {func_name} as specified") From 079d130effb45137430f8a5c6a76938effecc447 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Wed, 20 Sep 2023 10:50:33 -0700 Subject: [PATCH 3/4] fix up --- tests/conftest.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1771b98150..0d780f01cf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,7 +109,7 @@ def backend(request): if disable_backend: pytest.skip( - f"skipping {func_name} as the backend is disabled: {request.config.option.disable_backend}" + f"skipping {func_name} as the backend is disabled via --disable-backend" ) elif skip_backend: pytest.skip(f"skipping {func_name} as specified") @@ -127,10 +127,14 @@ def backend(request): pytest.mark.xfail(reason=f"expect {func_name} to fail as specified") ) + tensor_config, optimizer_config = request.param + + tensor = getattr(pyhf.tensor, tensor_config[0])(**tensor_config[1]) + optimizer = getattr(pyhf.optimize, optimizer_config[0])(**optimizer_config[1]) # actual execution here, after all checks is done - pyhf.set_backend(*request.param) + pyhf.set_backend(tensor, optimizer) - yield request.param + yield (tensor, optimizer) @pytest.fixture( From 67ab857b9d77b020dd41baf2757a6ecd664d6a00 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Thu, 5 Oct 2023 07:23:46 -0700 Subject: [PATCH 4/4] Update tests/conftest.py Co-authored-by: Matthew Feickert --- tests/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0d780f01cf..2f969e3444 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,7 +109,13 @@ def backend(request): if disable_backend: pytest.skip( - f"skipping {func_name} as the backend is disabled via --disable-backend" + f"skipping {func_name} as the backend is disabled via " + + " ".join( + [ + f"--disable-backend {choice}" + for choice in request.config.option.disable_backend + ] + ) ) elif skip_backend: pytest.skip(f"skipping {func_name} as specified")