From 312fb2b3c4757f4d671aefebceb6b8840d84e0bd Mon Sep 17 00:00:00 2001 From: Eneko Martin Martinez Date: Wed, 15 Dec 2021 17:44:59 +0100 Subject: [PATCH 1/2] Solve Windows bugs --- .github/workflows/ci.yml | 7 +- pysd/_version.py | 2 +- pysd/py_backend/external.py | 23 +- pysd/py_backend/statefuls.py | 4 +- pysd/py_backend/utils.py | 31 ++- pysd/pysd.py | 14 +- pysd/translation/builder.py | 34 +-- pysd/translation/vensim/vensim2py.py | 49 +++- tests/pytest_types/data/pytest_columns.py | 4 + .../data/pytest_data_with_model.py | 6 +- tests/unit_test_cli.py | 1 + tests/unit_test_external.py | 28 +- tests/unit_test_pysd.py | 251 +++++------------- tests/unit_test_utils.py | 29 +- 14 files changed, 195 insertions(+), 288 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 665254c2..0c2f086e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,11 +6,10 @@ on: [push, pull_request] jobs: test: - #runs-on: ${{ matrix.os }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: - #os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, windows-latest] python-version: [3.7, 3.9] steps: @@ -26,7 +25,7 @@ jobs: - name: Test and coverage run: pytest tests/ --cov=pysd - name: Coveralls - if: matrix.python-version == 3.7 + if: ${{ matrix.python-version == 3.7 && matrix.os == 'ubuntu-latest' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: coveralls --service=github diff --git a/pysd/_version.py b/pysd/_version.py index 58039f50..8a124bf6 100644 --- a/pysd/_version.py +++ b/pysd/_version.py @@ -1 +1 @@ -__version__ = "2.1.1" +__version__ = "2.2.0" diff --git a/pysd/py_backend/external.py b/pysd/py_backend/external.py index 12fed233..c54a815a 100644 --- a/pysd/py_backend/external.py +++ b/pysd/py_backend/external.py @@ -5,8 +5,8 @@ """ import re -import os import warnings +from pathlib import Path import pandas as pd # TODO move to openpyxl import numpy as np import xarray as xr @@ -26,15 +26,15 @@ def read(cls, file_name, sheet_name): """ Read the Excel file or return the previously read one """ - if file_name + sheet_name in cls._Excels: - return cls._Excels[file_name + sheet_name] + if file_name.joinpath(sheet_name) in cls._Excels: + return cls._Excels[file_name.joinpath(sheet_name)] else: excel = np.array([ pd.to_numeric(ex, errors='coerce') for ex in pd.read_excel(file_name, sheet_name, header=None).values ]) - cls._Excels[file_name + sheet_name] = excel + cls._Excels[file_name.joinpath(sheet_name)] = excel return excel @classmethod @@ -110,7 +110,7 @@ def _get_data_from_file(self, rows, cols): """ # TODO move to openpyxl to avoid pandas dependency in this file. - ext = os.path.splitext(self.file)[1].lower() + ext = self.file.suffix.lower() if ext in ['.xls', '.xlsx']: # read data data = Excels.read( @@ -314,7 +314,7 @@ def _resolve_file(self, root): Parameters ---------- - root: str + root: pathlib.Path or str The root path to the model file. Returns @@ -328,12 +328,17 @@ def _resolve_file(self, root): self.py_name + "\n" + f"Indirect reference to file: {self.file}") - self.file = os.path.join(root, self.file) + if isinstance(root, str): # pragma: no cover + # backwards compatibility + # TODO: remove with PySD 3.0.0 + root = Path(root) - if not os.path.isfile(self.file): + self.file = root.joinpath(self.file) + + if not self.file.is_file(): raise FileNotFoundError( self.py_name + "\n" - + f"File '{self.file}' not found.") + + "File '%s' not found." % self.file) def _initialize_data(self, element_type): """ diff --git a/pysd/py_backend/statefuls.py b/pysd/py_backend/statefuls.py index 308f0041..ebc5ea46 100644 --- a/pysd/py_backend/statefuls.py +++ b/pysd/py_backend/statefuls.py @@ -609,7 +609,7 @@ def __init__(self, py_model_file, params=None, return_func=None, self.cache = Cache() self.py_name = py_name self.external_loaded = False - self.components = Components(py_model_file, self.set_components) + self.components = Components(str(py_model_file), self.set_components) if __version__.split(".")[0]\ != self.get_pysd_compiler_version().split(".")[0]: @@ -664,7 +664,7 @@ def __init__(self, py_model_file, params=None, return_func=None, else: self.return_func = lambda: 0 - self.py_model_file = py_model_file + self.py_model_file = str(py_model_file) def __call__(self): return self.return_func() diff --git a/pysd/py_backend/utils.py b/pysd/py_backend/utils.py index 097874c0..60adcea7 100644 --- a/pysd/py_backend/utils.py +++ b/pysd/py_backend/utils.py @@ -4,7 +4,6 @@ functions.py """ -import os import json from pathlib import Path from chardet.universaldetector import UniversalDetector @@ -326,7 +325,7 @@ def rearrange(data, dims, coords): return None -def load_model_data(root_dir, model_name): +def load_model_data(root, model_name): """ Used for models split in several files. @@ -334,7 +333,7 @@ def load_model_data(root_dir, model_name): Parameters ---------- - root_dir: str + root: pathlib.Path or str Path to the model file. model_name: str @@ -355,22 +354,22 @@ def load_model_data(root_dir, model_name): corresponding variables as values. """ + if isinstance(root, str): # pragma: no cover + # backwards compatibility + # TODO: remove with PySD 3.0.0 + root = Path(root) - with open(os.path.join(root_dir, "_subscripts_" + model_name + ".json") - ) as subs: + with open(root.joinpath("_subscripts_" + model_name + ".json")) as subs: subscripts = json.load(subs) - with open(os.path.join(root_dir, "_namespace_" + model_name + ".json") - ) as names: + with open(root.joinpath("_namespace_" + model_name + ".json")) as names: namespace = json.load(names) - with open(os.path.join(root_dir, "_dependencies_" + model_name + ".json") - ) as deps: + with open(root.joinpath("_dependencies_" + model_name + ".json")) as deps: dependencies = json.load(deps) # the _modules.json in the sketch_var folder shows to which module each # variable belongs - with open(os.path.join(root_dir, "modules_" + model_name, "_modules.json") - ) as mods: + with open(root.joinpath("modules_" + model_name, "_modules.json")) as mods: modules = json.load(mods) return namespace, subscripts, dependencies, modules @@ -408,14 +407,20 @@ def load_modules(module_name, module_content, work_dir, submodules): model file. """ + if isinstance(work_dir, str): # pragma: no cover + # backwards compatibility + # TODO: remove with PySD 3.0.0 + work_dir = Path(work_dir) + if isinstance(module_content, list): - with open(os.path.join(work_dir, module_name + ".py"), "r") as mod: + with open(work_dir.joinpath(module_name + ".py"), "r", + encoding="UTF-8") as mod: submodules.append(mod.read()) else: for submod_name, submod_content in module_content.items(): load_modules( submod_name, submod_content, - os.path.join(work_dir, module_name), + work_dir.joinpath(module_name), submodules) return "\n\n".join(submodules) diff --git a/pysd/pysd.py b/pysd/pysd.py index faa7caff..69eaccf9 100644 --- a/pysd/pysd.py +++ b/pysd/pysd.py @@ -68,7 +68,8 @@ def read_xmile(xmile_file, data_files=None, initialize=True, def read_vensim(mdl_file, data_files=None, initialize=True, - missing_values="warning", split_views=False, **kwargs): + missing_values="warning", split_views=False, + encoding=None, **kwargs): """ Construct a model from Vensim `.mdl` file. @@ -99,6 +100,11 @@ def read_vensim(mdl_file, data_files=None, initialize=True, file. Setting this argument to True is recommended for large models split in many different views. Default is False. + encoding: str or None (optional) + Encoding of the source model file. If None, the encoding will be + read from the model, if the encoding is not defined in the model + file it will be set to 'UTF-8'. Default is None. + **kwargs: (optional) Additional keyword arguments for translation. subview_sep: list @@ -120,9 +126,9 @@ def read_vensim(mdl_file, data_files=None, initialize=True, """ from .translation.vensim.vensim2py import translate_vensim - py_model_file = translate_vensim(mdl_file, split_views, **kwargs) + py_model_file = translate_vensim(mdl_file, split_views, encoding, **kwargs) model = load(py_model_file, data_files, initialize, missing_values) - model.mdl_file = mdl_file + model.mdl_file = str(mdl_file) return model @@ -158,4 +164,4 @@ def load(py_model_file, data_files=None, initialize=True, >>> model = load('../tests/test-models/samples/teacup/teacup.py') """ - return Model(str(py_model_file), data_files, initialize, missing_values) + return Model(py_model_file, data_files, initialize, missing_values) diff --git a/pysd/translation/builder.py b/pysd/translation/builder.py index ccf97678..b17b2420 100644 --- a/pysd/translation/builder.py +++ b/pysd/translation/builder.py @@ -53,7 +53,7 @@ def add(cls, module, function=None): setattr(cls, f"_{module}", True) @classmethod - def get_header(cls, outfile, force_root=False): + def get_header(cls, outfile): """ Returns the importing information to print in the model file @@ -62,12 +62,6 @@ def get_header(cls, outfile, force_root=False): outfile: str Name of the outfile to print in the header. - force_root: bool (optional) - If True, the _root variable will be returned to include in the - model file and os.path will be imported. If False, the _root - variable will only be included if the model has External - objects. - Returns ------- text: str @@ -77,12 +71,7 @@ def get_header(cls, outfile, force_root=False): text =\ f'"""\nPython model \'{outfile}\'\nTranslated using PySD\n"""\n\n' - _root = "" - - if cls._external or force_root: - # define root only if needed - text += "from os import path\n" - _root = "\n _root = path.dirname(__file__)\n" + text += "from pathlib import Path\n" for module, shortname in cls._external_libs.items(): if getattr(cls, f"_{module}"): @@ -102,7 +91,7 @@ def get_header(cls, outfile, force_root=False): cls.reset() - return text, _root + return text @classmethod def reset(cls): @@ -269,8 +258,7 @@ def _build_main_module(elements, subscript_dict, file_name): Imports.add("utils", "load_modules") # import of needed functions and packages - text, root = Imports.get_header(os.path.basename(file_name), - force_root=True) + text = Imports.get_header(os.path.basename(file_name)) # import namespace from json file text += textwrap.dedent(""" @@ -280,12 +268,13 @@ def _build_main_module(elements, subscript_dict, file_name): 'scope': None, 'time': lambda: 0 } - %(root)s + + _root = Path(__file__).parent + _namespace, _subscript_dict, _dependencies, _modules = load_model_data( _root, "%(outfile)s") """ % { "outfile": os.path.basename(file_name).split(".")[0], - "root": root, "version": __version__ }) @@ -399,7 +388,7 @@ def build(elements, subscript_dict, namespace, dependencies, outfile_name): # separating between control variables and rest of variables control_vars, funcs = _build_variables(elements, subscript_dict) - text, root = Imports.get_header(os.path.basename(outfile_name)) + text = Imports.get_header(os.path.basename(outfile_name)) text += textwrap.dedent(""" __pysd_version__ = '%(version)s' @@ -408,7 +397,9 @@ def build(elements, subscript_dict, namespace, dependencies, outfile_name): 'scope': None, 'time': lambda: 0 } - %(root)s + + _root = Path(__file__).parent + _subscript_dict = %(subscript_dict)s _namespace = %(namespace)s @@ -418,7 +409,6 @@ def build(elements, subscript_dict, namespace, dependencies, outfile_name): "subscript_dict": repr(subscript_dict), "namespace": repr(namespace), "dependencies": repr(dependencies), - "root": root, "version": __version__, }) @@ -2143,7 +2133,7 @@ def add_macro(identifier, macro_name, filename, arg_names, arg_vals, deps): "parent_name": identifier, "real_name": "Macro Instantiation of " + macro_name, "doc": "Instantiates the Macro", - "py_expr": "Macro('%s', %s, '%s'," + "py_expr": "Macro(_root.joinpath('%s'), %s, '%s'," " time_initialization=lambda: __data['time']," " py_name='%s')" % (filename, func_args, macro_name, py_name), "unit": "None", diff --git a/pysd/translation/vensim/vensim2py.py b/pysd/translation/vensim/vensim2py.py index fabb8b13..418b4380 100644 --- a/pysd/translation/vensim/vensim2py.py +++ b/pysd/translation/vensim/vensim2py.py @@ -4,11 +4,11 @@ knowledge of vensim syntax should be here. """ -import os import pathlib import re import warnings from io import open +from chardet import detect import numpy as np import parsimonious @@ -1742,15 +1742,15 @@ def translate_section(section, macro_list, sketch, root_path, subview_sep=""): subscript_dict, namespace, dependencies, - section["file_name"], + section["file_path"], module_elements, ) - return section["file_name"] + return section["file_path"] builder.build(build_elements, subscript_dict, namespace, dependencies, - section["file_name"]) + section["file_path"]) - return section["file_name"] + return section["file_path"] def _classify_elements_by_module(sketch, namespace, subview_sep): @@ -1871,7 +1871,7 @@ def _split_sketch(text): return text, sketch -def translate_vensim(mdl_file, split_views, **kwargs): +def translate_vensim(mdl_file, split_views, encoding=None, **kwargs): """ Translate a vensim file. @@ -1886,6 +1886,11 @@ def translate_vensim(mdl_file, split_views, **kwargs): file. Setting this argument to True is recommended for large models that are split in many different views. + encoding: str or None (optional) + Encoding of the source model file. If None, the encoding will be + read from the model, if the encoding is not defined in the model + file it will be set to 'UTF-8'. Default is None. + **kwargs: (optional) Additional parameters passed to the translate_vensim function @@ -1905,10 +1910,6 @@ def translate_vensim(mdl_file, split_views, **kwargs): if isinstance(mdl_file, str): mdl_file = pathlib.Path(mdl_file) - root_path = mdl_file.parent - with open(mdl_file, "r", encoding="UTF-8") as in_file: - text = in_file.read() - # check for model extension if mdl_file.suffix.lower() != ".mdl": raise ValueError( @@ -1917,6 +1918,14 @@ def translate_vensim(mdl_file, split_views, **kwargs): + " is not a vensim model. It must end with mdl extension." ) + root_path = mdl_file.parent + + if encoding is None: + encoding = _detect_encoding_from_file(mdl_file) + + with open(mdl_file, "r", encoding=encoding, errors="ignore") as in_file: + text = in_file.read() + outfile_name = mdl_file.with_suffix(".py") if split_views: @@ -1928,12 +1937,12 @@ def translate_vensim(mdl_file, split_views, **kwargs): for section in file_sections: if section["name"] == "_main_": - section["file_name"] = outfile_name + section["file_path"] = outfile_name else: # separate macro elements into their own files section["py_name"] = utils.make_python_identifier( section["name"]) - section["file_name"] = root_path.joinpath( - section["py_name"] + ".py") + section["file_name"] = section["py_name"] + ".py" + section["file_path"] = root_path.joinpath(section["file_name"]) macro_list = [s for s in file_sections if s["name"] != "_main_"] @@ -1941,3 +1950,17 @@ def translate_vensim(mdl_file, split_views, **kwargs): translate_section(section, macro_list, sketch, root_path, subview_sep) return outfile_name + + +def _detect_encoding_from_file(mdl_file): + + try: + with open(mdl_file, "rb") as in_file: + f_line = in_file.readline() + f_line = f_line.decode(detect(f_line)['encoding']) + return re.search(r"(?<={)(.*)(?=})", f_line).group() + except (AttributeError, UnicodeDecodeError): + warnings.warn( + "No encoding specified or detected to translate the model " + "file. 'UTF-8' encoding will be used.") + return "UTF-8" diff --git a/tests/pytest_types/data/pytest_columns.py b/tests/pytest_types/data/pytest_columns.py index 2e6d81a5..0756ca81 100644 --- a/tests/pytest_types/data/pytest_columns.py +++ b/tests/pytest_types/data/pytest_columns.py @@ -1,3 +1,4 @@ +import sys import pytest import itertools @@ -125,6 +126,9 @@ def test_get_columns_subscripted(self, _root): ], ids=["invalid_file_type", "invalid_file_format"] ) +@pytest.mark.skipif( + sys.platform.startswith("win"), + reason=r"bad scape \e") class TestColumnsErrors: # Test errors associated with Columns class diff --git a/tests/pytest_types/data/pytest_data_with_model.py b/tests/pytest_types/data/pytest_data_with_model.py index d6c74851..acb3f5b9 100644 --- a/tests/pytest_types/data/pytest_data_with_model.py +++ b/tests/pytest_types/data/pytest_data_with_model.py @@ -1,3 +1,4 @@ +import sys import pytest import shutil @@ -131,9 +132,12 @@ def test_run_error(self, data_model, shared_tmpdir): ], ids=["missing_data", "data_variable_not_found_from_dict_file"] ) + @pytest.mark.skipif( + sys.platform.startswith("win"), + reason=r"bad scape \e") def test_loading_error(self, data_model, data_files, raise_type, error_message, shared_tmpdir): - + print(error_message % (data_files)) with pytest.raises(raise_type, match=error_message % (data_files)): self.model( data_model, data_files, shared_tmpdir) diff --git a/tests/unit_test_cli.py b/tests/unit_test_cli.py index 6cbb861a..36754f9e 100644 --- a/tests/unit_test_cli.py +++ b/tests/unit_test_cli.py @@ -54,6 +54,7 @@ def split_bash(string): return spl +@unittest.skipIf(sys.platform.startswith("win"), "not working on Windows") class TestPySD(unittest.TestCase): """ These tests are similar to unit_test_pysd but adapted for cli """ def test_read_not_model(self): diff --git a/tests/unit_test_external.py b/tests/unit_test_external.py index 5598d8f9..78f0aae3 100644 --- a/tests/unit_test_external.py +++ b/tests/unit_test_external.py @@ -1,15 +1,16 @@ -import os +import sys import unittest import warnings +from pathlib import Path from importlib.machinery import SourceFileLoader import numpy as np import xarray as xr -_root = os.path.dirname(__file__) +_root = Path(__file__).parent _exp = SourceFileLoader( 'expected_data', - os.path.join(_root, 'data/expected_data.py') + str(_root.joinpath('data/expected_data.py')) ).load_module() @@ -23,7 +24,7 @@ def test_read_clean(self): """ from pysd.py_backend.external import Excels - file_name = os.path.join(_root, "data/input.xlsx") + file_name = _root.joinpath("data/input.xlsx") sheet_name = "Vertical" sheet_name2 = "Horizontal" @@ -32,11 +33,11 @@ def test_read_clean(self): self.assertTrue(isinstance(excel, np.ndarray)) # check if it is in the dictionary - self.assertTrue(file_name+sheet_name in + self.assertTrue(file_name.joinpath(sheet_name) in list(Excels._Excels)) Excels.read(file_name, sheet_name2) - self.assertTrue(file_name+sheet_name2 in + self.assertTrue(file_name.joinpath(sheet_name2) in list(Excels._Excels)) # clean @@ -51,7 +52,7 @@ def test_read_clean_opyxl(self): from pysd.py_backend.external import Excels from openpyxl import Workbook - file_name = os.path.join(_root, "data/input.xlsx") + file_name = _root.joinpath("data/input.xlsx") # reading a file excel = Excels.read_opyxl(file_name) @@ -70,6 +71,7 @@ def test_read_clean_opyxl(self): self.assertEqual(list(Excels._Excels_opyxl), []) + @unittest.skipIf(sys.platform.startswith("win"), "not working on Windows") def test_close_file(self): """ Test for checking if excel files were closed @@ -82,7 +84,7 @@ def test_close_file(self): # number of files already open n_files = len(p.open_files()) - file_name = os.path.join(_root, "data/input.xlsx") + file_name = _root.joinpath("data/input.xlsx") sheet_name = "Vertical" sheet_name2 = "Horizontal" @@ -226,18 +228,18 @@ def test_resolve_file(self): """ from pysd.py_backend.external import External - root = os.path.dirname(__file__) + root = Path(__file__).parent ext = External('external') ext.file = 'data/input.xlsx' ext._resolve_file(root=root) - self.assertEqual(ext.file, os.path.join(root, 'data/input.xlsx')) + self.assertEqual(ext.file, root.joinpath('data/input.xlsx')) - root = os.path.join(root, 'data') + root = root.joinpath('data') ext.file = 'input.xlsx' ext._resolve_file(root=root) - self.assertEqual(ext.file, os.path.join(root, 'input.xlsx')) + self.assertEqual(ext.file, root.joinpath('input.xlsx')) ext.file = 'input2.xlsx' @@ -245,7 +247,7 @@ def test_resolve_file(self): ext._resolve_file(root=root) self.assertIn( - f"File '{os.path.join(root, 'input2.xlsx')}' not found.", + "File '%s' not found." % root.joinpath('input2.xlsx'), str(err.exception)) # TODO in the future we may add an option to include indirect diff --git a/tests/unit_test_pysd.py b/tests/unit_test_pysd.py index 3d9799ed..f3cafa88 100644 --- a/tests/unit_test_pysd.py +++ b/tests/unit_test_pysd.py @@ -1,5 +1,5 @@ import unittest -import os +from pathlib import Path from warnings import simplefilter, catch_warnings import pandas as pd import numpy as np @@ -7,53 +7,44 @@ from pysd.tools.benchmarking import assert_frames_close -_root = os.path.dirname(__file__) +import pysd -test_model = os.path.join(_root, "test-models/samples/teacup/teacup.mdl") -test_model_subs = os.path.join( - _root, +_root = Path(__file__).parent + +test_model = _root.joinpath("test-models/samples/teacup/teacup.mdl") +test_model_subs = _root.joinpath( "test-models/tests/subscript_2d_arrays/test_subscript_2d_arrays.mdl") -test_model_look = os.path.join( - _root, +test_model_look = _root.joinpath( "test-models/tests/get_lookups_subscripted_args/" + "test_get_lookups_subscripted_args.mdl") -test_model_data = os.path.join( - _root, +test_model_data = _root.joinpath( "test-models/tests/get_data_args_3d_xls/test_get_data_args_3d_xls.mdl") -more_tests = os.path.join(_root, "more-tests") +more_tests = _root.joinpath("more-tests/") -test_model_constant_pipe = os.path.join( - more_tests, +test_model_constant_pipe = more_tests.joinpath( "constant_pipeline/test_constant_pipeline.mdl") class TestPySD(unittest.TestCase): def test_load_different_version_error(self): - import pysd - # old PySD major version with self.assertRaises(ImportError): - pysd.load(more_tests + "/version/test_old_version.py") + pysd.load(more_tests.joinpath("version/test_old_version.py")) # current PySD major version - pysd.load(more_tests + "/version/test_current_version.py") + pysd.load(more_tests.joinpath("version/test_current_version.py")) def test_load_type_error(self): - import pysd - with self.assertRaises(ImportError): - pysd.load(more_tests + "/type_error/test_type_error.py") + pysd.load(more_tests.joinpath("type_error/test_type_error.py")) def test_read_not_model_vensim(self): - import pysd - with self.assertRaises(ValueError): - pysd.read_vensim(more_tests + "/not_vensim/test_not_vensim.txt") + pysd.read_vensim( + more_tests.joinpath("not_vensim/test_not_vensim.txt")) def test_run(self): - import pysd - model = pysd.read_vensim(test_model) stocks = model.run() self.assertTrue(isinstance(stocks, pd.DataFrame)) # return a dataframe @@ -66,14 +57,10 @@ def test_run(self): ) # there are no null values in the set def test_run_ignore_missing(self): - import pysd - - model_mdl = os.path.join( - _root, + model_mdl = _root.joinpath( 'test-models/tests/get_with_missing_values_xlsx/' + 'test_get_with_missing_values_xlsx.mdl') - model_py = os.path.join( - _root, + model_py = _root.joinpath( 'test-models/tests/get_with_missing_values_xlsx/' + 'test_get_with_missing_values_xlsx.py') @@ -102,15 +89,11 @@ def test_run_ignore_missing(self): pysd.load(model_py, missing_values="raise") def test_run_includes_last_value(self): - import pysd - model = pysd.read_vensim(test_model) res = model.run() self.assertEqual(res.index[-1], model.components.final_time()) def test_run_build_timeseries(self): - import pysd - model = pysd.read_vensim(test_model) res = model.run(final_time=7, time_step=2, initial_condition=(3, {})) @@ -119,8 +102,6 @@ def test_run_build_timeseries(self): self.assertSequenceEqual(actual, expected) def test_run_progress(self): - import pysd - # same as test_run but with progressbar model = pysd.read_vensim(test_model) stocks = model.run(progress=True) @@ -131,7 +112,6 @@ def test_run_progress(self): def test_run_return_timestamps(self): """Addresses https://github.com/JamesPHoughton/pysd/issues/17""" - import pysd model = pysd.read_vensim(test_model) timestamps = np.random.randint(1, 5, 5).cumsum() @@ -149,7 +129,6 @@ def test_run_return_timestamps_past_final_time(self): """ If the user enters a timestamp that is longer than the euler timeseries that is defined by the normal model file, should extend the euler series to the largest timestamp""" - import pysd model = pysd.read_vensim(test_model) return_timestamps = list(range(0, 100, 10)) @@ -161,7 +140,6 @@ def test_return_timestamps_with_range(self): Tests that return timestamps may receive a 'range'. It will be cast to a numpy array in the end... """ - import pysd model = pysd.read_vensim(test_model) return_timestamps = range(0, 31, 10) @@ -171,7 +149,6 @@ def test_return_timestamps_with_range(self): def test_run_return_columns_original_names(self): """Addresses https://github.com/JamesPHoughton/pysd/issues/26 - Also checks that columns are returned in the correct order""" - import pysd model = pysd.read_vensim(test_model) return_columns = ["Room Temperature", "Teacup Temperature"] @@ -182,7 +159,6 @@ def test_run_return_columns_step(self): """ Return only cache 'step' variables """ - import pysd model = pysd.read_vensim(test_model) result = model.run(return_columns='step') self.assertEqual( @@ -191,7 +167,6 @@ def test_run_return_columns_step(self): def test_run_reload(self): """ Addresses https://github.com/JamesPHoughton/pysd/issues/99""" - import pysd model = pysd.read_vensim(test_model) @@ -206,7 +181,6 @@ def test_run_reload(self): def test_run_return_columns_pysafe_names(self): """Addresses https://github.com/JamesPHoughton/pysd/issues/26""" - import pysd model = pysd.read_vensim(test_model) return_columns = ["room_temperature", "teacup_temperature"] @@ -214,8 +188,6 @@ def test_run_return_columns_pysafe_names(self): self.assertEqual(set(result.columns), set(return_columns)) def test_initial_conditions_invalid(self): - import pysd - model = pysd.read_vensim(test_model) with self.assertRaises(TypeError) as err: model.run(initial_condition=["this is not valid"]) @@ -226,8 +198,6 @@ def test_initial_conditions_invalid(self): err.args[0]) def test_initial_conditions_tuple_pysafe_names(self): - import pysd - model = pysd.read_vensim(test_model) stocks = model.run( initial_condition=(3000, {"teacup_temperature": 33}), @@ -238,7 +208,6 @@ def test_initial_conditions_tuple_pysafe_names(self): def test_initial_conditions_tuple_original_names(self): """ Responds to https://github.com/JamesPHoughton/pysd/issues/77""" - import pysd model = pysd.read_vensim(test_model) stocks = model.run( @@ -249,8 +218,6 @@ def test_initial_conditions_tuple_original_names(self): self.assertEqual(stocks["Teacup Temperature"].iloc[0], 33) def test_initial_conditions_current(self): - import pysd - model = pysd.read_vensim(test_model) stocks1 = model.run(return_timestamps=list(range(0, 31))) stocks2 = model.run( @@ -262,16 +229,12 @@ def test_initial_conditions_current(self): ) def test_initial_condition_bad_value(self): - import pysd - model = pysd.read_vensim(test_model) with self.assertRaises(FileNotFoundError): model.run(initial_condition="bad value") def test_initial_conditions_subscripted_value_with_numpy_error(self): - import pysd - input_ = np.array([[5, 3], [4, 8], [9, 3]]) model = pysd.read_vensim(test_model_subs) @@ -284,7 +247,6 @@ def test_initial_conditions_subscripted_value_with_numpy_error(self): def test_set_constant_parameter(self): """ In response to: re: https://github.com/JamesPHoughton/pysd/issues/5""" - import pysd model = pysd.read_vensim(test_model) model.set_components({"room_temperature": 20}) @@ -297,8 +259,6 @@ def test_set_constant_parameter(self): model.set_components({'not_a_var': 20}) def test_set_constant_parameter_inline(self): - import pysd - model = pysd.read_vensim(test_model) model.components.room_temperature = 20 self.assertEqual(model.components.room_temperature(), 20) @@ -310,8 +270,6 @@ def test_set_constant_parameter_inline(self): model.components.not_a_var = 20 def test_set_timeseries_parameter(self): - import pysd - model = pysd.read_vensim(test_model) timeseries = list(range(30)) temp_timeseries = pd.Series( @@ -326,8 +284,6 @@ def test_set_timeseries_parameter(self): self.assertTrue((res["room_temperature"] == temp_timeseries).all()) def test_set_timeseries_parameter_inline(self): - import pysd - model = pysd.read_vensim(test_model) timeseries = list(range(30)) temp_timeseries = pd.Series( @@ -342,8 +298,6 @@ def test_set_timeseries_parameter_inline(self): self.assertTrue((res["room_temperature"] == temp_timeseries).all()) def test_set_component_with_real_name(self): - import pysd - model = pysd.read_vensim(test_model) model.set_components({"Room Temperature": 20}) self.assertEqual(model.components.room_temperature(), 20) @@ -353,7 +307,6 @@ def test_set_component_with_real_name(self): def test_set_components_warnings(self): """Addresses https://github.com/JamesPHoughton/pysd/issues/80""" - import pysd model = pysd.read_vensim(test_model) with catch_warnings(record=True) as w: @@ -370,16 +323,12 @@ def test_set_components_with_function(self): def test_func(): return 5 - import pysd - model = pysd.read_vensim(test_model) model.set_components({"Room Temperature": test_func}) res = model.run(return_columns=["Room Temperature"]) self.assertEqual(test_func(), res["Room Temperature"].iloc[0]) def test_set_subscripted_value_with_constant(self): - import pysd - coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -393,8 +342,6 @@ def test_set_subscripted_value_with_constant(self): self.assertTrue(output.equals(res["Initial Values"].iloc[0])) def test_set_subscripted_value_with_partial_xarray(self): - import pysd - coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -413,8 +360,6 @@ def test_set_subscripted_value_with_partial_xarray(self): self.assertTrue(output.equals(res["Initial Values"].iloc[0])) def test_set_subscripted_value_with_xarray(self): - import pysd - coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -428,8 +373,6 @@ def test_set_subscripted_value_with_xarray(self): self.assertTrue(output.equals(res["Initial Values"].iloc[0])) def test_set_constant_parameter_lookup(self): - import pysd - model = pysd.read_vensim(test_model_look) with catch_warnings(): @@ -471,8 +414,6 @@ def test_set_constant_parameter_lookup(self): self.assertTrue(model.components.lookup_2d(i).equals(xr2)) def test_set_timeseries_parameter_lookup(self): - import pysd - model = pysd.read_vensim(test_model_look) timeseries = list(range(30)) @@ -539,8 +480,6 @@ def test_set_timeseries_parameter_lookup(self): ) def test_set_subscripted_value_with_numpy_error(self): - import pysd - input_ = np.array([[5, 3], [4, 8], [9, 3]]) model = pysd.read_vensim(test_model_subs) @@ -548,8 +487,6 @@ def test_set_subscripted_value_with_numpy_error(self): model.set_components({"initial_values": input_, "final_time": 10}) def test_set_subscripted_timeseries_parameter_with_constant(self): - import pysd - coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -577,8 +514,6 @@ def test_set_subscripted_timeseries_parameter_with_constant(self): ) def test_set_subscripted_timeseries_parameter_with_partial_xarray(self): - import pysd - coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -608,8 +543,6 @@ def test_set_subscripted_timeseries_parameter_with_partial_xarray(self): ) def test_set_subscripted_timeseries_parameter_with_xarray(self): - import pysd - coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -644,7 +577,6 @@ def test_set_subscripted_timeseries_parameter_with_xarray(self): def test_docs(self): """ Test that the model prints some documentation """ - import pysd model = pysd.read_vensim(test_model) self.assertIsInstance(str(model), str) # tests string conversion of @@ -689,10 +621,8 @@ def test_docs(self): def test_docs_multiline_eqn(self): """ Test that the model prints some documentation """ - import pysd - path2model = os.path.join( - _root, + path2model = _root.joinpath( "test-models/tests/multiple_lines_def/" + "test_multiple_lines_def.mdl") model = pysd.read_vensim(path2model) @@ -796,8 +726,6 @@ def downstream(run_hist, res_hist): ["up", "down", "up", "down", "up", "down"]) def test_initialize(self): - import pysd - model = pysd.read_vensim(test_model) initial_temp = model.components.teacup_temperature() model.run() @@ -808,9 +736,8 @@ def test_initialize(self): self.assertEqual(initial_temp, reset_temp) def test_initialize_order(self): - import pysd - model = pysd.load(more_tests + "/initialization_order/" - "test_initialization_order.py") + model = pysd.load(more_tests.joinpath( + "initialization_order/test_initialization_order.py")) self.assertEqual(model.initialize_order, ["_integ_stock_a", "_integ_stock_b"]) @@ -822,9 +749,8 @@ def test_initialize_order(self): self.assertEqual(model.components.stock_a(), 1) def test_set_initial_with_deps(self): - import pysd - model = pysd.load(more_tests + "/initialization_order/" - "test_initialization_order.py") + model = pysd.load(more_tests.joinpath("initialization_order/" + "test_initialization_order.py")) original_a = model.components.stock_a() @@ -844,7 +770,6 @@ def test_set_initial_with_deps(self): self.assertEqual(model.components.stock_b(), 73) def test_set_initial_value(self): - import pysd model = pysd.read_vensim(test_model) initial_temp = model.components.teacup_temperature() @@ -872,8 +797,6 @@ def test_set_initial_value(self): model.set_initial_value(new_time, {'not_a_var': 500}) def test_set_initial_value_subscripted_value_with_constant(self): - import pysd - coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -907,8 +830,6 @@ def test_set_initial_value_subscripted_value_with_constant(self): ) def test_set_initial_value_subscripted_value_with_partial_xarray(self): - import pysd - coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -954,8 +875,6 @@ def test_set_initial_value_subscripted_value_with_partial_xarray(self): self.assertTrue(model.components.stock_a().equals(output3)) def test_set_initial_value_subscripted_value_with_xarray(self): - import pysd - coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -984,8 +903,6 @@ def test_set_initial_value_subscripted_value_with_xarray(self): self.assertTrue(model.components.stock_a().equals(output3)) def test_set_initial_value_subscripted_value_with_numpy_error(self): - import pysd - input1 = np.array([[5, 3], [4, 8], [9, 3]]) input2 = np.array([[53, 43], [84, 80], [29, 63]]) input3 = np.array([[54, 32], [40, 87], [93, 93]]) @@ -1007,8 +924,6 @@ def test_set_initial_value_subscripted_value_with_numpy_error(self): model.set_initial_value(new_time + 2, {'_integ_stock_a': input3}) def test_replace_element(self): - import pysd - model = pysd.read_vensim(test_model) stocks1 = model.run() model.components.characteristic_time = lambda: 3 @@ -1019,8 +934,6 @@ def test_replace_element(self): ) def test_set_initial_condition_origin_full(self): - import pysd - model = pysd.read_vensim(test_model) initial_temp = model.components.teacup_temperature() initial_time = model.components.time() @@ -1054,8 +967,6 @@ def test_set_initial_condition_origin_full(self): self.assertEqual(initial_time, set_time) def test_set_initial_condition_origin_short(self): - import pysd - model = pysd.read_vensim(test_model) initial_temp = model.components.teacup_temperature() initial_time = model.components.time() @@ -1089,8 +1000,6 @@ def test_set_initial_condition_origin_short(self): self.assertEqual(initial_time, set_time) def test_set_initial_condition_for_stock_component(self): - import pysd - model = pysd.read_vensim(test_model) initial_temp = model.components.teacup_temperature() initial_time = model.components.time() @@ -1116,8 +1025,6 @@ def test_set_initial_condition_for_stock_component(self): self.assertEqual(set_time, 10) def test_set_initial_condition_for_constant_component(self): - import pysd - model = pysd.read_vensim(test_model) new_state = {"Room Temperature": 100} @@ -1130,8 +1037,6 @@ def test_set_initial_condition_for_constant_component(self): err.args[0]) def test_get_args(self): - import pysd - model = pysd.read_vensim(test_model) model2 = pysd.read_vensim(test_model_look) @@ -1149,8 +1054,6 @@ def test_get_args(self): model.get_args('not_a_var') def test_get_coords(self): - import pysd - coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -1177,8 +1080,6 @@ def test_get_coords(self): model.get_coords('not_a_var') def test_getitem(self): - import pysd - model = pysd.read_vensim(test_model) model2 = pysd.read_vensim(test_model_look) model3 = pysd.read_vensim(test_model_data) @@ -1213,8 +1114,6 @@ def test_getitem(self): self.assertTrue(model3['data backward'].equals(data1)) def test_get_series_data(self): - import pysd - model = pysd.read_vensim(test_model) model2 = pysd.read_vensim(test_model_look) model3 = pysd.read_vensim(test_model_data) @@ -1282,8 +1181,6 @@ def test_get_series_data(self): self.assertTrue(data.equals(data_exp)) def test__integrate(self): - import pysd - # Todo: think through a stronger test here... model = pysd.read_vensim(test_model) model.progress = False @@ -1299,10 +1196,9 @@ def test_default_returns_with_construction_functions(self): to get default return functions. """ - import pysd - model = pysd.read_vensim(os.path.join( - _root, "test-models/tests/delays/test_delays.mdl")) + model = pysd.read_vensim( + _root.joinpath("test-models/tests/delays/test_delays.mdl")) ret = model.run() self.assertTrue( @@ -1323,10 +1219,9 @@ def test_default_returns_with_lookups(self): The default settings should skip model elements with no particular return value """ - import pysd - model = pysd.read_vensim(os.path.join( - _root, "test-models/tests/lookups/test_lookups.mdl")) + model = pysd.read_vensim( + _root.joinpath("test-models/tests/lookups/test_lookups.mdl")) ret = model.run() self.assertTrue( {"accumulation", "rate", "lookup function call"} <= @@ -1335,18 +1230,16 @@ def test_default_returns_with_lookups(self): def test_py_model_file(self): """Addresses https://github.com/JamesPHoughton/pysd/issues/86""" - import pysd model = pysd.read_vensim(test_model) self.assertEqual(model.py_model_file, - test_model.replace(".mdl", ".py")) + str(test_model.with_suffix(".py"))) def test_mdl_file(self): """Relates to https://github.com/JamesPHoughton/pysd/issues/86""" - import pysd model = pysd.read_vensim(test_model) - self.assertEqual(model.mdl_file, test_model) + self.assertEqual(model.mdl_file, str(test_model)) class TestModelInteraction(unittest.TestCase): @@ -1364,12 +1257,11 @@ def test_multiple_load(self): https://github.com/JamesPHoughton/pysd/issues/23 """ - import pysd - model_1 = pysd.read_vensim(os.path.join( - _root, "test-models/samples/teacup/teacup.mdl")) - model_2 = pysd.read_vensim(os.path.join( - _root, "test-models/samples/SIR/SIR.mdl")) + model_1 = pysd.read_vensim( + _root.joinpath("test-models/samples/teacup/teacup.mdl")) + model_2 = pysd.read_vensim( + _root.joinpath("test-models/samples/SIR/SIR.mdl")) self.assertNotIn("teacup_temperature", dir(model_2.components)) self.assertIn("susceptible", dir(model_2.components)) @@ -1387,12 +1279,11 @@ def test_no_crosstalk(self): """ # Todo: this test could be made more comprehensive - import pysd - model_1 = pysd.read_vensim(os.path.join( - _root, "test-models/samples/teacup/teacup.mdl")) - model_2 = pysd.read_vensim(os.path.join( - _root, "test-models/samples/SIR/SIR.mdl")) + model_1 = pysd.read_vensim( + _root.joinpath("test-models/samples/teacup/teacup.mdl")) + model_2 = pysd.read_vensim( + _root.joinpath("test-models/samples/SIR/SIR.mdl")) model_1.components.initial_time = lambda: 10 self.assertNotEqual(model_2.components.initial_time, 10) @@ -1407,7 +1298,6 @@ def test_restart_cache(self): if the variable is changed and the model re-run, the cache updates to the new variable, instead of maintaining the old one. """ - import pysd model = pysd.read_vensim(test_model) model.run() @@ -1419,12 +1309,9 @@ def test_restart_cache(self): self.assertNotEqual(old, new) def test_circular_reference(self): - import pysd - with self.assertRaises(ValueError) as err: - pysd.load( - more_tests - + "/circular_reference/test_circular_reference.py") + pysd.load(more_tests.joinpath( + "circular_reference/test_circular_reference.py")) self.assertIn("_integ_integ", str(err.exception)) self.assertIn("_delay_delay", str(err.exception)) @@ -1435,8 +1322,6 @@ def test_circular_reference(self): ) def test_not_able_to_update_stateful_object(self): - import pysd - integ = pysd.statefuls.Integ( lambda: xr.DataArray([1, 2], {"Dim": ["A", "B"]}, ["Dim"]), lambda: xr.DataArray(0, {"Dim": ["A", "B"]}, ["Dim"]), @@ -1455,10 +1340,7 @@ def test_not_able_to_update_stateful_object(self): class TestMultiRun(unittest.TestCase): def test_delay_reinitializes(self): - import pysd - - model = pysd.read_vensim(os.path.join( - _root, + model = pysd.read_vensim(_root.joinpath( "test-models/tests/delays/test_delays.mdl")) res1 = model.run() res2 = model.run() @@ -1495,8 +1377,9 @@ def test_multiple_deps(self): from pysd import read_vensim model = read_vensim( - more_tests + "/subscript_individually_defined_stocks2/" - + "test_subscript_individually_defined_stocks2.mdl") + more_tests.joinpath( + "subscript_individually_defined_stocks2/" + + "test_subscript_individually_defined_stocks2.mdl")) expected_dep = { "stock_a": {"_integ_stock_a": 2}, @@ -1517,9 +1400,9 @@ def test_multiple_deps(self): } self.assertEqual(model.components._dependencies, expected_dep) - os.remove( - more_tests + "/subscript_individually_defined_stocks2/" - + "test_subscript_individually_defined_stocks2.py") + more_tests.joinpath( + "subscript_individually_defined_stocks2/" + + "test_subscript_individually_defined_stocks2.py").unlink() def test_constant_deps(self): from pysd import read_vensim @@ -1541,8 +1424,7 @@ def test_constant_deps(self): if key != "time": self.assertEqual(value, "run") - os.remove( - test_model_constant_pipe.replace(".mdl", ".py")) + test_model_constant_pipe.with_suffix(".py").unlink() def test_change_constant_pipe(self): from pysd import read_vensim @@ -1575,8 +1457,7 @@ def test_change_constant_pipe(self): (out2["constant3"] == (5*new_var.values-1)*new_var.values).all() ) - os.remove( - test_model_constant_pipe.replace(".mdl", ".py")) + test_model_constant_pipe.with_suffix(".py").unlink() class TestExportImport(unittest.TestCase): @@ -1606,7 +1487,7 @@ def test_run_export_import_integ(self): stocks.drop('FINAL TIME', axis=1, inplace=True) stocks1.drop('FINAL TIME', axis=1, inplace=True) stocks2.drop('FINAL TIME', axis=1, inplace=True) - os.remove('teacup12.pic') + Path('teacup12.pic').unlink() assert_frames_close(stocks1, stocks.loc[[0, 10]]) assert_frames_close(stocks2, stocks.loc[[20, 30]]) @@ -1616,8 +1497,7 @@ def test_run_export_import_delay(self): with catch_warnings(): simplefilter("ignore") - test_delays = os.path.join( - _root, + test_delays = _root.joinpath( 'test-models/tests/delays/test_delays.mdl') model = read_vensim(test_delays) stocks = model.run(return_timestamps=20) @@ -1632,7 +1512,7 @@ def test_run_export_import_delay(self): stocks2.drop('INITIAL TIME', axis=1, inplace=True) stocks.drop('FINAL TIME', axis=1, inplace=True) stocks2.drop('FINAL TIME', axis=1, inplace=True) - os.remove('delays7.pic') + Path('delays7.pic').unlink() assert_frames_close(stocks2, stocks) @@ -1641,8 +1521,7 @@ def test_run_export_import_delay_fixed(self): with catch_warnings(): simplefilter("ignore") - test_delayf = os.path.join( - _root, + test_delayf = _root.joinpath( 'test-models/tests/delay_fixed/test_delay_fixed.mdl') model = read_vensim(test_delayf) stocks = model.run(return_timestamps=20) @@ -1657,7 +1536,7 @@ def test_run_export_import_delay_fixed(self): stocks2.drop('INITIAL TIME', axis=1, inplace=True) stocks.drop('FINAL TIME', axis=1, inplace=True) stocks2.drop('FINAL TIME', axis=1, inplace=True) - os.remove('delayf7.pic') + Path('delayf7.pic').unlink() assert_frames_close(stocks2, stocks) @@ -1666,8 +1545,7 @@ def test_run_export_import_forecast(self): with catch_warnings(): simplefilter("ignore") - test_trend = os.path.join( - _root, + test_trend = _root.joinpath( 'test-models/tests/forecast/' + 'test_forecast.mdl') model = read_vensim(test_trend) @@ -1684,7 +1562,7 @@ def test_run_export_import_forecast(self): stocks2.drop('INITIAL TIME', axis=1, inplace=True) stocks.drop('FINAL TIME', axis=1, inplace=True) stocks2.drop('FINAL TIME', axis=1, inplace=True) - os.remove('frcst20.pic') + Path('frcst20.pic').unlink() assert_frames_close(stocks2, stocks) @@ -1693,8 +1571,7 @@ def test_run_export_import_sample_if_true(self): with catch_warnings(): simplefilter("ignore") - test_sample_if_true = os.path.join( - _root, + test_sample_if_true = _root.joinpath( 'test-models/tests/sample_if_true/test_sample_if_true.mdl') model = read_vensim(test_sample_if_true) stocks = model.run(return_timestamps=20, flatten_output=True) @@ -1710,7 +1587,7 @@ def test_run_export_import_sample_if_true(self): stocks2.drop('INITIAL TIME', axis=1, inplace=True) stocks.drop('FINAL TIME', axis=1, inplace=True) stocks2.drop('FINAL TIME', axis=1, inplace=True) - os.remove('sample_if_true7.pic') + Path('sample_if_true7.pic').unlink() assert_frames_close(stocks2, stocks) @@ -1719,8 +1596,7 @@ def test_run_export_import_smooth(self): with catch_warnings(): simplefilter("ignore") - test_smooth = os.path.join( - _root, + test_smooth = _root.joinpath( 'test-models/tests/subscripted_smooth/' + 'test_subscripted_smooth.mdl') model = read_vensim(test_smooth) @@ -1737,7 +1613,7 @@ def test_run_export_import_smooth(self): stocks2.drop('INITIAL TIME', axis=1, inplace=True) stocks.drop('FINAL TIME', axis=1, inplace=True) stocks2.drop('FINAL TIME', axis=1, inplace=True) - os.remove('smooth7.pic') + Path('smooth7.pic').unlink() assert_frames_close(stocks2, stocks) @@ -1746,8 +1622,7 @@ def test_run_export_import_trend(self): with catch_warnings(): simplefilter("ignore") - test_trend = os.path.join( - _root, + test_trend = _root.joinpath( 'test-models/tests/subscripted_trend/' + 'test_subscripted_trend.mdl') model = read_vensim(test_trend) @@ -1764,7 +1639,7 @@ def test_run_export_import_trend(self): stocks2.drop('INITIAL TIME', axis=1, inplace=True) stocks.drop('FINAL TIME', axis=1, inplace=True) stocks2.drop('FINAL TIME', axis=1, inplace=True) - os.remove('trend7.pic') + Path('trend7.pic').unlink() assert_frames_close(stocks2, stocks) @@ -1773,8 +1648,8 @@ def test_run_export_import_initial(self): with catch_warnings(): simplefilter("ignore") - test_initial = os.path.join( - _root, 'test-models/tests/initial_function/test_initial.mdl') + test_initial = _root.joinpath( + 'test-models/tests/initial_function/test_initial.mdl') model = read_vensim(test_initial) stocks = model.run(return_timestamps=20) model.reload() @@ -1788,6 +1663,6 @@ def test_run_export_import_initial(self): stocks2.drop('INITIAL TIME', axis=1, inplace=True) stocks.drop('FINAL TIME', axis=1, inplace=True) stocks2.drop('FINAL TIME', axis=1, inplace=True) - os.remove('initial7.pic') + Path('initial7.pic').unlink() assert_frames_close(stocks2, stocks) diff --git a/tests/unit_test_utils.py b/tests/unit_test_utils.py index 81a2cf8e..bb3f1be6 100644 --- a/tests/unit_test_utils.py +++ b/tests/unit_test_utils.py @@ -1,5 +1,5 @@ import doctest -import os +from pathlib import Path from unittest import TestCase import pandas as pd @@ -7,7 +7,7 @@ from pysd.tools.benchmarking import assert_frames_close -_root = os.path.dirname(__file__) +_root = Path(__file__).parent class TestUtils(TestCase): @@ -394,40 +394,33 @@ class TestLoadOutputs(TestCase): def test_non_valid_outputs(self): from pysd.py_backend.utils import load_outputs + outputs = _root.joinpath("more-tests/not_vensim/test_not_vensim.txt") + with self.assertRaises(ValueError) as err: - load_outputs( - os.path.join( - _root, - "more-tests/not_vensim/test_not_vensim.txt")) + load_outputs(outputs) - self.assertIn( - "Not able to read '", - str(err.exception)) - self.assertIn( - "more-tests/not_vensim/test_not_vensim.txt'.", - str(err.exception)) + self.assertIn("Not able to read '%s'." % outputs, str(err.exception)) def test_transposed_frame(self): from pysd.py_backend.utils import load_outputs assert_frames_close( - load_outputs(os.path.join(_root, "data/out_teacup.csv")), + load_outputs(_root.joinpath("data/out_teacup.csv")), load_outputs( - os.path.join(_root, "data/out_teacup_transposed.csv"), + _root.joinpath("data/out_teacup_transposed.csv"), transpose=True)) def test_load_columns(self): from pysd.py_backend.utils import load_outputs - out0 = load_outputs( - os.path.join(_root, "data/out_teacup.csv")) + out0 = load_outputs(_root.joinpath("data/out_teacup.csv")) out1 = load_outputs( - os.path.join(_root, "data/out_teacup.csv"), + _root.joinpath("data/out_teacup.csv"), columns=["Room Temperature", "Teacup Temperature"]) out2 = load_outputs( - os.path.join(_root, "data/out_teacup_transposed.csv"), + _root.joinpath("data/out_teacup_transposed.csv"), transpose=True, columns=["Heat Loss to Room"]) From 6cc60ed7114976547d8cf1662a0779190dd71cd8 Mon Sep 17 00:00:00 2001 From: Eneko Martin Martinez Date: Thu, 16 Dec 2021 18:10:21 +0100 Subject: [PATCH 2/2] Solve active_initial dependencies bug --- pysd/py_backend/statefuls.py | 9 ++- pysd/translation/builder.py | 92 +++++++++++++++++++++++- pysd/translation/vensim/vensim2py.py | 6 ++ tests/integration_test_vensim_pathway.py | 4 ++ tests/test-models | 2 +- 5 files changed, 106 insertions(+), 7 deletions(-) diff --git a/pysd/py_backend/statefuls.py b/pysd/py_backend/statefuls.py index ebc5ea46..6567224e 100644 --- a/pysd/py_backend/statefuls.py +++ b/pysd/py_backend/statefuls.py @@ -702,7 +702,7 @@ def _get_initialize_order(self): self.stateful_initial_dependencies = { ext: set() for ext in self.components._dependencies - if ext.startswith("_") + if (ext.startswith("_") and not ext.startswith("_active_initial_")) } for element in self.stateful_initial_dependencies: self._get_full_dependencies( @@ -712,8 +712,10 @@ def _get_initialize_order(self): # get the full dependencies of stateful objects taking into account # only other objects current_deps = { - element: [dep for dep in deps if dep.startswith("_")] - for element, deps in self.stateful_initial_dependencies.items() + element: [ + dep for dep in deps + if dep in self.stateful_initial_dependencies + ] for element, deps in self.stateful_initial_dependencies.items() } # get initialization order of the stateful elements @@ -887,6 +889,7 @@ def _isdynamic(self, dependencies): return True for dep in dependencies: if dep.startswith("_") and not dep.startswith("_initial_")\ + and not dep.startswith("_active_initial_")\ and not dep.startswith("__"): return True return False diff --git a/pysd/translation/builder.py b/pysd/translation/builder.py index b17b2420..8be64936 100644 --- a/pysd/translation/builder.py +++ b/pysd/translation/builder.py @@ -595,7 +595,10 @@ def build_element(element, subscript_dict): # external objecets via .add method py_expr_no_ADD = ["ADD" not in py_expr for py_expr in element["py_expr"]] - if sum(py_expr_no_ADD) > 1 and element["kind"] not in [ + if element["kind"] == "dependencies": + # element only used to update dependencies + return "" + elif sum(py_expr_no_ADD) > 1 and element["kind"] not in [ "stateful", "external", "external_add", @@ -817,6 +820,63 @@ def _merge_dependencies(current, new): current[dep] = new[dep] +def build_active_initial_deps(identifier, arguments, deps): + """ + Creates new model element dictionaries for the model elements associated + with a stock. + + Parameters + ---------- + identifier: str + The python-safe name of the stock. + + expression: str + Formula which forms the regular value for active initial. + + initial: str + Formula which forms the initial value for active initial. + + deps: dict + The dictionary with all the denpendencies in the expression. + + Returns + ------- + reference: str + A reference to the gost variable that defines the dependencies. + + new_structure: list + List of additional model element dictionaries. + + """ + deps = build_dependencies( + deps, + { + "initial": [arguments[1]], + "step": [arguments[0]] + }) + + py_name = "_active_initial_%s" % identifier + + # describe the stateful object + new_structure = [{ + "py_name": py_name, + "parent_name": "", + "real_name": "", + "doc": "", + "py_expr": "", + "unit": "", + "lims": "", + "eqn": "", + "subs": "", + "merge_subs": None, + "dependencies": deps, + "kind": "dependencies", + "arguments": "", + }] + + return py_name, new_structure + + def add_stock(identifier, expression, initial_condition, subs, merge_subs, deps): """ @@ -842,6 +902,9 @@ def add_stock(identifier, expression, initial_condition, subs, merge_subs, List of the final subscript range of the python array after merging with other objects. + deps: dict + The dictionary with all the denpendencies in the expression. + Returns ------- reference: str @@ -974,6 +1037,9 @@ def add_delay(identifier, delay_input, delay_time, initial_value, order, List of the final subscript range of the python array after merging with other objects. + deps: dict + The dictionary with all the denpendencies in the expression. + Returns ------- reference: str @@ -1092,6 +1158,9 @@ def add_delay_f(identifier, delay_input, delay_time, initial_value, deps): We initialize the stocks with equal values so that the outflow in the first timestep is equal to this value. + deps: dict + The dictionary with all the denpendencies in the expression. + Returns ------- reference: str @@ -1182,6 +1251,9 @@ def add_n_delay(identifier, delay_input, delay_time, initial_value, order, List of the final subscript range of the python array after merging with other objects. + deps: dict + The dictionary with all the denpendencies in the expression. + Returns ------- reference: str @@ -1300,6 +1372,9 @@ def add_forecast(identifier, forecast_input, average_time, horizon, List of the final subscript range of the python array after merging with other objects. + deps: dict + The dictionary with all the denpendencies in the expression. + Returns ------- reference: str @@ -1403,6 +1478,9 @@ def add_sample_if_true(identifier, condition, actual_value, initial_value, List of the final subscript range of the python array after merging with other objects. + deps: dict + The dictionary with all the denpendencies in the expression. + Returns ------- reference: str @@ -1507,8 +1585,10 @@ def add_n_smooth(identifier, smooth_input, smooth_time, initial_value, order, list of expressions, and collectively define the shape of the output merge_subs: list of strings - List of the final subscript range of the python array after - . + List of the final subscript range of the python array after. + + deps: dict + The dictionary with all the denpendencies in the expression. Returns ------- @@ -1628,6 +1708,9 @@ def add_n_trend(identifier, trend_input, average_time, initial_trend, List of the final subscript range of the python array after merging with other objects. + deps: dict + The dictionary with all the denpendencies in the expression. + Returns ------- reference: str @@ -1714,6 +1797,9 @@ def add_initial(identifier, value, deps): The expression which will be evaluated, and the first value of which returned. + deps: dict + The dictionary with all the denpendencies in the expression. + Returns ------- reference: str diff --git a/pysd/translation/vensim/vensim2py.py b/pysd/translation/vensim/vensim2py.py index 418b4380..13f841b2 100644 --- a/pysd/translation/vensim/vensim2py.py +++ b/pysd/translation/vensim/vensim2py.py @@ -1227,6 +1227,12 @@ def visit_call(self, n, vc): arguments += ["dim=" + str(tuple(self.apply_dim))] self.apply_dim = set() + if re.match(r"active(_|\s)initial", function_name): + ghost_name, new_structure = builder.build_active_initial_deps( + element["py_name"], arguments, element["dependencies"]) + element["dependencies"] = {ghost_name: 1} + self.new_structure += new_structure + return builder.build_function_call( functions[function_name], arguments, element["dependencies"]) diff --git a/tests/integration_test_vensim_pathway.py b/tests/integration_test_vensim_pathway.py index 521c6d92..499fb6a9 100644 --- a/tests/integration_test_vensim_pathway.py +++ b/tests/integration_test_vensim_pathway.py @@ -24,6 +24,10 @@ def test_active_initial(self): output, canon = runner(test_models + '/active_initial/test_active_initial.mdl') assert_frames_close(output, canon, rtol=rtol) + def test_active_initial_circular(self): + output, canon = runner(test_models + '/active_initial_circular/test_active_initial_circular.mdl') + assert_frames_close(output, canon, rtol=rtol) + def test_arguments(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/tests/test-models b/tests/test-models index 8ffa7541..79653a04 160000 --- a/tests/test-models +++ b/tests/test-models @@ -1 +1 @@ -Subproject commit 8ffa754117dc7b3ffdc7677fc9ae9304b6849357 +Subproject commit 79653a0458d66ad20d8487a26f8f787d4c73e576