From a86bf9ce005a379fff6c2f6ab1cd79c32957414a Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez Date: Fri, 22 Dec 2023 16:42:25 +0100 Subject: [PATCH] Make integration tests filter only specific warnings --- docs/whats_new.rst | 6 +- pysd/builders/python/python_model_builder.py | 4 +- tests/conftest.py | 9 ++ .../pytest_integration_vensim_pathway.py | 123 ++++++++++++++---- .../pytest_integration_xmile_pathway.py | 19 ++- 5 files changed, 125 insertions(+), 36 deletions(-) diff --git a/docs/whats_new.rst b/docs/whats_new.rst index 86891b9f..f1078273 100644 --- a/docs/whats_new.rst +++ b/docs/whats_new.rst @@ -25,9 +25,9 @@ Performance Internal Changes ~~~~~~~~~~~~~~~~ -- Move old :py:meth:`pysd.py_backend.model.Macro.set_components` to :py:meth:`pysd.py_backend.model.Macro._set_components`, and create new method with the same name without the `new` argument. - -- Move old :py:meth:`pysd.py_backend.model.Macro.set_stateful` to :py:meth:`pysd.py_backend.model.Macro._set_stateful`. +- Move old :py:meth:`pysd.py_backend.model.Macro.set_components` to :py:meth:`pysd.py_backend.model.Macro._set_components`, and create new method with the same name without the `new` argument. (`@enekomartinmartinez `_) +- Move old :py:meth:`pysd.py_backend.model.Macro.set_stateful` to :py:meth:`pysd.py_backend.model.Macro._set_stateful`. (`@enekomartinmartinez `_) +- Make integration tests filter only specific warnings. (`@enekomartinmartinez `_) v3.12.0 (2023/10/02) -------------------- diff --git a/pysd/builders/python/python_model_builder.py b/pysd/builders/python/python_model_builder.py index ce432835..019a69c5 100644 --- a/pysd/builders/python/python_model_builder.py +++ b/pysd/builders/python/python_model_builder.py @@ -663,7 +663,7 @@ def build_element(self) -> None: if ", " in self.type: warn( f"Variable '{self.name}' is defined with different types:" - f" '{self.type}'. This may cause bugs when trying to" + f" '{self.type}'. This may cause bugs when trying to " "change its value or applying other methods from the " "pysd.py_backend.model.Model class. Running the model " "without modifying this variable should not cause any " @@ -672,7 +672,7 @@ def build_element(self) -> None: elif ", " in self.subtype: warn( f"Variable '{self.name}' is defined with different subtypes:" - f" '{self.subtype}'. This may cause bugs when trying to" + f" '{self.subtype}'. This may cause bugs when trying to " "change its value or applying other methods from the " "pysd.py_backend.model.Model class. Running the model " "without modifying this variable should not cause any " diff --git a/tests/conftest.py b/tests/conftest.py index fe3f39d2..0cbe8b5b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,3 +46,12 @@ def model(_root, tmp_path, model_path): return load(new_path) else: return ValueError("Invalid model") + + +@pytest.fixture(scope="session") +def ignore_warns(): + # warnings to be ignored in the integration tests + return [ + "numpy.ndarray size changed, may indicate binary incompatibility.", + "Creating an ndarray from ragged nested sequences.*" + ] diff --git a/tests/pytest_integration/pytest_integration_vensim_pathway.py b/tests/pytest_integration/pytest_integration_vensim_pathway.py index a4afd463..de24aac3 100644 --- a/tests/pytest_integration/pytest_integration_vensim_pathway.py +++ b/tests/pytest_integration/pytest_integration_vensim_pathway.py @@ -1,10 +1,9 @@ -import warnings +import re import shutil import pytest from pysd.tools.benchmarking import runner, assert_frames_close -# TODO add warnings catcher per test vensim_test = { "abs": { @@ -22,7 +21,10 @@ "allocate_available": { "folder": "allocate_available", "file": "test_allocate_available.mdl", - "rtol": 2e-2 + "rtol": 2e-2, + "warns": [ + "Variable '.*' is defined with different .*types: '.*'", + ] }, "allocate_by_priority": { "folder": "allocate_by_priority", @@ -39,7 +41,10 @@ "arguments": { "folder": "arguments", "file": "test_arguments.mdl", - "rtol": 1e-2 # TODO test why it is failing with smaller tolerance + "rtol": 1e-2, # TODO test why it is failing with smaller tolerance + "warns": [ + "_delay.*\nDelay time very small, casting delay order from .*" + ] }, "array_with_line_break": { "folder": "array_with_line_break", @@ -76,7 +81,10 @@ }, "delay_fixed": { "folder": "delay_fixed", - "file": "test_delay_fixed.mdl" + "file": "test_delay_fixed.mdl", + "warns": [ + ".*\nCasting delay order from.*" + ] }, "delay_numeric_error": { "folder": "delay_numeric_error", @@ -88,7 +96,11 @@ }, "delay_pipeline": { "folder": "delay_pipeline", - "file": "test_pipeline_delays.mdl" + "file": "test_pipeline_delays.mdl", + "warns": [ + "_delay.*\nDelay time very small, casting delay order from .*", + ".*\nCasting delay order from.*" + ] }, "delays": { "folder": "delays", @@ -116,7 +128,10 @@ }, "except_subranges": { "folder": "except_subranges", - "file": "test_except_subranges.mdl" + "file": "test_except_subranges.mdl", + "warns": [ + "Variable '.*' is defined with different .*types: '.*'", + ] }, "exp": { "folder": "exp", @@ -148,7 +163,10 @@ }, marks=pytest.mark.xfail(reason="csv files not implemented")), "get_constants_subranges": { "folder": "get_constants_subranges", - "file": "test_get_constants_subranges.mdl" + "file": "test_get_constants_subranges.mdl", + "warns": [ + "Variable '.*' is defined with different .*types: '.*'", + ] }, "get_data": pytest.param({ "folder": "get_data", @@ -164,7 +182,10 @@ }, "get_lookups_subscripted_args": { "folder": "get_lookups_subscripted_args", - "file": "test_get_lookups_subscripted_args.mdl" + "file": "test_get_lookups_subscripted_args.mdl", + "warns": [ + "_ext.*\nextrapolating data .* the m.*mum value of the .*", + ] }, "get_lookups_subset": { "folder": "get_lookups_subset", @@ -172,7 +193,11 @@ }, "get_mixed_definitions": { "folder": "get_mixed_definitions", - "file": "test_get_mixed_definitions.mdl" + "file": "test_get_mixed_definitions.mdl", + "warns": [ + "Variable '.*' is defined with different .*types: '.*'", + "_ext.*\nextrapolating data .* the m.*mum value of the .*", + ] }, "get_subscript_3d_arrays_xls": { "folder": "get_subscript_3d_arrays_xls", @@ -188,7 +213,11 @@ }, "get_with_missing_values_xlsx": { "folder": "get_with_missing_values_xlsx", - "file": "test_get_with_missing_values_xlsx.mdl" + "file": "test_get_with_missing_values_xlsx.mdl", + "warns": [ + "_ext.*\nD.* value missing or non-valid in:.*", + "_ext.*\nextrapolating data .* the m.*mum value of the .*", + ] }, "get_xls_cellrange": { "folder": "get_xls_cellrange", @@ -308,7 +337,11 @@ }, "odd_number_quotes": { "folder": "odd_number_quotes", - "file": "teacup_3quotes.mdl" + "file": "teacup_3quotes.mdl", + "warns": [ + "No encoding specified or detected to translate the model " + "file. 'UTF-8' encoding will be used." + ] }, "parentheses": { "folder": "parentheses", @@ -316,7 +349,10 @@ }, "partial_range_definitions": { "folder": "partial_range_definitions", - "file": "test_partial_range_definitions.mdl" + "file": "test_partial_range_definitions.mdl", + "warns": [ + "\nDimension given by subscripts.*" + ] }, "power": { "folder": "power", @@ -324,7 +360,11 @@ }, "reality_checks": { "folder": "reality_checks", - "file": "test_reality_checks.mdl" + "file": "test_reality_checks.mdl", + "warns": [ + "':.*:' detected. The expression content is not parsed and " + "will be ignored.", + ] }, "reference_capitalization": { "folder": "reference_capitalization", @@ -332,7 +372,10 @@ }, "repeated_subscript": { "folder": "repeated_subscript", - "file": "test_repeated_subscript.mdl" + "file": "test_repeated_subscript.mdl", + "warns": [ + "\nAdding new subscript range to subscript_dict:.*" + ] }, "rounding": { "folder": "rounding", @@ -344,7 +387,12 @@ }, "smaller_range": { "folder": "smaller_range", - "file": "test_smaller_range.mdl" + "file": "test_smaller_range.mdl", + "warns": [ + "\nDimension given by subscripts.*", + "\nAdding new subscript range to subscript_dict:.*", + "Variable '.*' is defined with different .*types: '.*'", + ] }, "smooth": { "folder": "smooth", @@ -364,7 +412,10 @@ }, "subrange_merge": { "folder": "subrange_merge", - "file": "test_subrange_merge.mdl" + "file": "test_subrange_merge.mdl", + "warns": [ + "Variable '.*' is defined with different .*types: '.*'", + ] }, "subscript_1d_arrays": { "folder": "subscript_1d_arrays", @@ -444,7 +495,10 @@ }, "subscript_mixed_assembly": { "folder": "subscript_mixed_assembly", - "file": "test_subscript_mixed_assembly.mdl" + "file": "test_subscript_mixed_assembly.mdl", + "warns": [ + "Variable '.*' is defined with different .*types: '.*'", + ] }, "subscript_multiples": { "folder": "subscript_multiples", @@ -496,7 +550,10 @@ }, "subscripted_lookups": { "folder": "subscripted_lookups", - "file": "test_subscripted_lookups.mdl" + "file": "test_subscripted_lookups.mdl", + "warns": [ + "_.*\nextrapolating data .* the m.*mum value of the .*", + ] }, "subscripted_ramp_step": { "folder": "subscripted_ramp_step", @@ -504,7 +561,10 @@ }, "subscripted_round": { "folder": "subscripted_round", - "file": "test_subscripted_round.mdl" + "file": "test_subscripted_round.mdl", + "warns": [ + "Variable '.*' is defined with different .*types: '.*'", + ] }, "subscripted_smooth": { "folder": "subscripted_smooth", @@ -560,7 +620,11 @@ }, "vector_select": { "folder": "vector_select", - "file": "test_vector_select.mdl" + "file": "test_vector_select.mdl", + "warns": [ + "Vensim's help says that numerical_action=5 computes the " + "product of selection_array . expression_array..*" + ] }, "with_lookup": { "folder": "with_lookup", @@ -623,6 +687,11 @@ def data_path(self, test_folder, test_data): else: return None + @pytest.fixture + def warns(self, test_data, ignore_warns): + ign_warns = test_data.get('warns', []) + ignore_warns + return [re.compile(w) for w in ign_warns] + @pytest.fixture def kwargs(self, test_data): """Fixture for atol and rtol""" @@ -633,8 +702,12 @@ def kwargs(self, test_data): kwargs["rtol"] = test_data["rtol"] return kwargs - def test_read_vensim_file(self, model_path, data_path, kwargs): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - output, canon = runner(model_path, data_files=data_path) + def test_read_vensim_file(self, model_path, data_path, kwargs, + recwarn, warns): + output, canon = runner(model_path, data_files=data_path) + for warn in recwarn: + warn = str(warn.message) + assert any([re.match(pwarn, warn) for pwarn in warns]), \ + f"Couldn't match warning:\n{warn}" + assert_frames_close(output, canon, verbose=True, **kwargs) diff --git a/tests/pytest_integration/pytest_integration_xmile_pathway.py b/tests/pytest_integration/pytest_integration_xmile_pathway.py index d452a441..f2eef392 100644 --- a/tests/pytest_integration/pytest_integration_xmile_pathway.py +++ b/tests/pytest_integration/pytest_integration_xmile_pathway.py @@ -1,10 +1,9 @@ -import warnings +import re import shutil import pytest from pysd.tools.benchmarking import runner, assert_frames_close -# TODO add warnings catcher per test xmile_test = { "abs": { @@ -278,6 +277,11 @@ def model_path(self, test_folder, test_data): """Return model path""" return test_folder.joinpath(test_data["file"]) + @pytest.fixture + def warns(self, test_data, ignore_warns): + ign_warns = test_data.get('warns', []) + ignore_warns + return [re.compile(w) for w in ign_warns] + @pytest.fixture def kwargs(self, test_data): """Fixture for atol and rtol""" @@ -288,8 +292,11 @@ def kwargs(self, test_data): kwargs["rtol"] = test_data["rtol"] return kwargs - def test_read_vensim_file(self, model_path, kwargs): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - output, canon = runner(model_path) + def test_read_xmile_file(self, model_path, kwargs, recwarn, warns): + output, canon = runner(model_path) + for warn in recwarn: + warn = str(warn.message) + assert any([re.match(pwarn, warn) for pwarn in warns]), \ + f"Couldn't match warning:\n{warn}" + assert_frames_close(output, canon, **kwargs)