Make integration tests filter only specific warnings

SDXorg · Dec 22, 2023 · a86bf9c · a86bf9c
1 parent 0a25e04
commit a86bf9c
Show file tree

Hide file tree

Showing 5 changed files with 125 additions and 36 deletions.
diff --git a/docs/whats_new.rst b/docs/whats_new.rst
@@ -25,9 +25,9 @@ Performance
 
 Internal Changes
 ~~~~~~~~~~~~~~~~
-- Move old :py:meth:`pysd.py_backend.model.Macro.set_components` to :py:meth:`pysd.py_backend.model.Macro._set_components`, and create new method with the same name without the `new` argument.
-
-- Move old :py:meth:`pysd.py_backend.model.Macro.set_stateful` to :py:meth:`pysd.py_backend.model.Macro._set_stateful`.
+- Move old :py:meth:`pysd.py_backend.model.Macro.set_components` to :py:meth:`pysd.py_backend.model.Macro._set_components`, and create new method with the same name without the `new` argument. (`@enekomartinmartinez <https://github.com/enekomartinmartinez>`_)
+- Move old :py:meth:`pysd.py_backend.model.Macro.set_stateful` to :py:meth:`pysd.py_backend.model.Macro._set_stateful`. (`@enekomartinmartinez <https://github.com/enekomartinmartinez>`_)
+- Make integration tests filter only specific warnings. (`@enekomartinmartinez <https://github.com/enekomartinmartinez>`_)
 
 v3.12.0 (2023/10/02)
 --------------------

diff --git a/pysd/builders/python/python_model_builder.py b/pysd/builders/python/python_model_builder.py
@@ -663,7 +663,7 @@ def build_element(self) -> None:
         if ", " in self.type:
             warn(
                 f"Variable '{self.name}' is defined with different types:"
-                f" '{self.type}'. This may cause bugs when trying to"
+                f" '{self.type}'. This may cause bugs when trying to "
                 "change its value or applying other methods from the "
                 "pysd.py_backend.model.Model class. Running the model "
                 "without modifying this variable should not cause any "
@@ -672,7 +672,7 @@ def build_element(self) -> None:
         elif ", " in self.subtype:
             warn(
                 f"Variable '{self.name}' is defined with different subtypes:"
-                f" '{self.subtype}'. This may cause bugs when trying to"
+                f" '{self.subtype}'. This may cause bugs when trying to "
                 "change its value or applying other methods from the "
                 "pysd.py_backend.model.Model class. Running the model "
                 "without modifying this variable should not cause any "

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -46,3 +46,12 @@ def model(_root, tmp_path, model_path):
         return load(new_path)
     else:
         return ValueError("Invalid model")
+
+
+@pytest.fixture(scope="session")
+def ignore_warns():
+    # warnings to be ignored in the integration tests
+    return [
+        "numpy.ndarray size changed, may indicate binary incompatibility.",
+        "Creating an ndarray from ragged nested sequences.*"
+    ]
diff --git a/tests/pytest_integration/pytest_integration_vensim_pathway.py b/tests/pytest_integration/pytest_integration_vensim_pathway.py
@@ -1,10 +1,9 @@
-import warnings
+import re
 import shutil
 import pytest
 
 from pysd.tools.benchmarking import runner, assert_frames_close
 
-# TODO add warnings catcher per test
 
 vensim_test = {
     "abs": {
@@ -22,7 +21,10 @@
     "allocate_available": {
         "folder": "allocate_available",
         "file": "test_allocate_available.mdl",
-        "rtol": 2e-2
+        "rtol": 2e-2,
+        "warns": [
+            "Variable '.*' is defined with different .*types: '.*'",
+        ]
     },
     "allocate_by_priority": {
         "folder": "allocate_by_priority",
@@ -39,7 +41,10 @@
     "arguments": {
         "folder": "arguments",
         "file": "test_arguments.mdl",
-        "rtol": 1e-2  # TODO test why it is failing with smaller tolerance
+        "rtol": 1e-2,  # TODO test why it is failing with smaller tolerance
+        "warns": [
+            "_delay.*\nDelay time very small, casting delay order from .*"
+        ]
     },
     "array_with_line_break": {
         "folder": "array_with_line_break",
@@ -76,7 +81,10 @@
     },
     "delay_fixed": {
         "folder": "delay_fixed",
-        "file": "test_delay_fixed.mdl"
+        "file": "test_delay_fixed.mdl",
+        "warns": [
+            ".*\nCasting delay order from.*"
+        ]
     },
     "delay_numeric_error": {
         "folder": "delay_numeric_error",
@@ -88,7 +96,11 @@
     },
     "delay_pipeline": {
         "folder": "delay_pipeline",
-        "file": "test_pipeline_delays.mdl"
+        "file": "test_pipeline_delays.mdl",
+        "warns": [
+            "_delay.*\nDelay time very small, casting delay order from .*",
+            ".*\nCasting delay order from.*"
+        ]
     },
     "delays": {
         "folder": "delays",
@@ -116,7 +128,10 @@
     },
     "except_subranges": {
         "folder": "except_subranges",
-        "file": "test_except_subranges.mdl"
+        "file": "test_except_subranges.mdl",
+        "warns": [
+            "Variable '.*' is defined with different .*types: '.*'",
+        ]
     },
     "exp": {
         "folder": "exp",
@@ -148,7 +163,10 @@
     },  marks=pytest.mark.xfail(reason="csv files not implemented")),
     "get_constants_subranges": {
         "folder": "get_constants_subranges",
-        "file": "test_get_constants_subranges.mdl"
+        "file": "test_get_constants_subranges.mdl",
+        "warns": [
+            "Variable '.*' is defined with different .*types: '.*'",
+        ]
     },
     "get_data": pytest.param({
         "folder": "get_data",
@@ -164,15 +182,22 @@
     },
     "get_lookups_subscripted_args": {
         "folder": "get_lookups_subscripted_args",
-        "file": "test_get_lookups_subscripted_args.mdl"
+        "file": "test_get_lookups_subscripted_args.mdl",
+        "warns": [
+            "_ext.*\nextrapolating data .* the m.*mum value of the .*",
+        ]
     },
     "get_lookups_subset": {
         "folder": "get_lookups_subset",
         "file": "test_get_lookups_subset.mdl"
     },
     "get_mixed_definitions": {
         "folder": "get_mixed_definitions",
-        "file": "test_get_mixed_definitions.mdl"
+        "file": "test_get_mixed_definitions.mdl",
+        "warns": [
+            "Variable '.*' is defined with different .*types: '.*'",
+            "_ext.*\nextrapolating data .* the m.*mum value of the .*",
+        ]
     },
     "get_subscript_3d_arrays_xls": {
         "folder": "get_subscript_3d_arrays_xls",
@@ -188,7 +213,11 @@
     },
     "get_with_missing_values_xlsx": {
         "folder": "get_with_missing_values_xlsx",
-        "file": "test_get_with_missing_values_xlsx.mdl"
+        "file": "test_get_with_missing_values_xlsx.mdl",
+        "warns": [
+            "_ext.*\nD.* value missing or non-valid in:.*",
+            "_ext.*\nextrapolating data .* the m.*mum value of the .*",
+        ]
     },
     "get_xls_cellrange": {
         "folder": "get_xls_cellrange",
@@ -308,31 +337,45 @@
     },
     "odd_number_quotes": {
         "folder": "odd_number_quotes",
-        "file": "teacup_3quotes.mdl"
+        "file": "teacup_3quotes.mdl",
+        "warns": [
+            "No encoding specified or detected to translate the model "
+            "file. 'UTF-8' encoding will be used."
+        ]
     },
     "parentheses": {
         "folder": "parentheses",
         "file": "test_parens.mdl"
     },
     "partial_range_definitions": {
         "folder": "partial_range_definitions",
-        "file": "test_partial_range_definitions.mdl"
+        "file": "test_partial_range_definitions.mdl",
+        "warns": [
+            "\nDimension given by subscripts.*"
+        ]
     },
     "power": {
         "folder": "power",
         "file": "power.mdl"
     },
     "reality_checks": {
         "folder": "reality_checks",
-        "file": "test_reality_checks.mdl"
+        "file": "test_reality_checks.mdl",
+        "warns": [
+            "':.*:' detected. The expression content is not parsed and "
+            "will be ignored.",
+        ]
     },
     "reference_capitalization": {
         "folder": "reference_capitalization",
         "file": "test_reference_capitalization.mdl"
     },
     "repeated_subscript": {
         "folder": "repeated_subscript",
-        "file": "test_repeated_subscript.mdl"
+        "file": "test_repeated_subscript.mdl",
+        "warns": [
+            "\nAdding new subscript range to subscript_dict:.*"
+        ]
     },
     "rounding": {
         "folder": "rounding",
@@ -344,7 +387,12 @@
     },
     "smaller_range": {
         "folder": "smaller_range",
-        "file": "test_smaller_range.mdl"
+        "file": "test_smaller_range.mdl",
+        "warns": [
+            "\nDimension given by subscripts.*",
+            "\nAdding new subscript range to subscript_dict:.*",
+            "Variable '.*' is defined with different .*types: '.*'",
+        ]
     },
     "smooth": {
         "folder": "smooth",
@@ -364,7 +412,10 @@
     },
     "subrange_merge": {
         "folder": "subrange_merge",
-        "file": "test_subrange_merge.mdl"
+        "file": "test_subrange_merge.mdl",
+        "warns": [
+            "Variable '.*' is defined with different .*types: '.*'",
+        ]
     },
     "subscript_1d_arrays": {
         "folder": "subscript_1d_arrays",
@@ -444,7 +495,10 @@
     },
     "subscript_mixed_assembly": {
         "folder": "subscript_mixed_assembly",
-        "file": "test_subscript_mixed_assembly.mdl"
+        "file": "test_subscript_mixed_assembly.mdl",
+        "warns": [
+            "Variable '.*' is defined with different .*types: '.*'",
+        ]
     },
     "subscript_multiples": {
         "folder": "subscript_multiples",
@@ -496,15 +550,21 @@
     },
     "subscripted_lookups": {
         "folder": "subscripted_lookups",
-        "file": "test_subscripted_lookups.mdl"
+        "file": "test_subscripted_lookups.mdl",
+        "warns": [
+            "_.*\nextrapolating data .* the m.*mum value of the .*",
+        ]
     },
     "subscripted_ramp_step": {
         "folder": "subscripted_ramp_step",
         "file": "test_subscripted_ramp_step.mdl"
     },
     "subscripted_round": {
         "folder": "subscripted_round",
-        "file": "test_subscripted_round.mdl"
+        "file": "test_subscripted_round.mdl",
+        "warns": [
+            "Variable '.*' is defined with different .*types: '.*'",
+        ]
     },
     "subscripted_smooth": {
         "folder": "subscripted_smooth",
@@ -560,7 +620,11 @@
     },
     "vector_select": {
         "folder": "vector_select",
-        "file": "test_vector_select.mdl"
+        "file": "test_vector_select.mdl",
+        "warns": [
+            "Vensim's help says that numerical_action=5 computes the "
+            "product of selection_array . expression_array..*"
+        ]
     },
     "with_lookup": {
         "folder": "with_lookup",
@@ -623,6 +687,11 @@ def data_path(self, test_folder, test_data):
         else:
             return None
 
+    @pytest.fixture
+    def warns(self, test_data, ignore_warns):
+        ign_warns = test_data.get('warns', []) + ignore_warns
+        return [re.compile(w) for w in ign_warns]
+
     @pytest.fixture
     def kwargs(self, test_data):
         """Fixture for atol and rtol"""
@@ -633,8 +702,12 @@ def kwargs(self, test_data):
             kwargs["rtol"] = test_data["rtol"]
         return kwargs
 
-    def test_read_vensim_file(self, model_path, data_path, kwargs):
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            output, canon = runner(model_path, data_files=data_path)
+    def test_read_vensim_file(self, model_path, data_path, kwargs,
+                              recwarn, warns):
+        output, canon = runner(model_path, data_files=data_path)
+        for warn in recwarn:
+            warn = str(warn.message)
+            assert any([re.match(pwarn, warn) for pwarn in warns]), \
+                f"Couldn't match warning:\n{warn}"
+
         assert_frames_close(output, canon, verbose=True, **kwargs)
diff --git a/tests/pytest_integration/pytest_integration_xmile_pathway.py b/tests/pytest_integration/pytest_integration_xmile_pathway.py
@@ -1,10 +1,9 @@
-import warnings
+import re
 import shutil
 import pytest
 
 from pysd.tools.benchmarking import runner, assert_frames_close
 
-# TODO add warnings catcher per test
 
 xmile_test = {
     "abs": {
@@ -278,6 +277,11 @@ def model_path(self, test_folder, test_data):
         """Return model path"""
         return test_folder.joinpath(test_data["file"])
 
+    @pytest.fixture
+    def warns(self, test_data, ignore_warns):
+        ign_warns = test_data.get('warns', []) + ignore_warns
+        return [re.compile(w) for w in ign_warns]
+
     @pytest.fixture
     def kwargs(self, test_data):
         """Fixture for atol and rtol"""
@@ -288,8 +292,11 @@ def kwargs(self, test_data):
             kwargs["rtol"] = test_data["rtol"]
         return kwargs
 
-    def test_read_vensim_file(self, model_path, kwargs):
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            output, canon = runner(model_path)
+    def test_read_xmile_file(self, model_path, kwargs, recwarn, warns):
+        output, canon = runner(model_path)
+        for warn in recwarn:
+            warn = str(warn.message)
+            assert any([re.match(pwarn, warn) for pwarn in warns]), \
+                f"Couldn't match warning:\n{warn}"
+
         assert_frames_close(output, canon, **kwargs)