From 00efc5fecfe1286387d99a780a0db09b81172975 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez Date: Tue, 6 Sep 2022 21:28:08 +0200 Subject: [PATCH 01/43] Make PySD work with parsimonius 0.10.0 --- docs/whats_new.rst | 26 +++++++++++++++++++ pysd/_version.py | 2 +- .../parsing_grammars/element_object.peg | 2 +- requirements.txt | 2 +- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/docs/whats_new.rst b/docs/whats_new.rst index f59c4729..48f22f10 100644 --- a/docs/whats_new.rst +++ b/docs/whats_new.rst @@ -1,5 +1,31 @@ What's New ========== +v3.7.0 (to be released) +------------------- + +New Features +~~~~~~~~~~~~ + +Breaking changes +~~~~~~~~~~~~~~~~ + +Deprecations +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ + +Documentation +~~~~~~~~~~~~~ + +Performance +~~~~~~~~~~~ + +Internal Changes +~~~~~~~~~~~~~~~~ +- Make PySD work with :py:mod:`parsimonius` 0.10.0. + + v3.6.1 (2022/09/05) ------------------- diff --git a/pysd/_version.py b/pysd/_version.py index b202327a..46f67e7f 100644 --- a/pysd/_version.py +++ b/pysd/_version.py @@ -1 +1 @@ -__version__ = "3.6.1" +__version__ = "3.7.0" diff --git a/pysd/translators/vensim/parsing_grammars/element_object.peg b/pysd/translators/vensim/parsing_grammars/element_object.peg index 49256520..b0af013a 100644 --- a/pysd/translators/vensim/parsing_grammars/element_object.peg +++ b/pysd/translators/vensim/parsing_grammars/element_object.peg @@ -29,7 +29,7 @@ subscript_copy = name _ "<->" _ name_mapping # Subscript mapping subscript_mapping_list = "->" _ subscript_mapping _ ("," _ subscript_mapping _)* -subscript_mapping = (_ name_mapping _) / (_ "(" _ name_mapping _ ":" _ index_list _")" ) +subscript_mapping = (_ name_mapping _) / (_ "(" _ name_mapping _ ":" _ index_list _ ")" ) name_mapping = basic_id / escape_group # Subscript except match diff --git a/requirements.txt b/requirements.txt index dfa1b357..e0947fec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ pandas -parsimonious==0.9.0 +parsimonious xarray xlrd lxml From f5aec83ef16b6f5f9224087496d1082b58305102 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez Date: Tue, 6 Sep 2022 22:28:16 +0200 Subject: [PATCH 02/43] Fix bug when a WITH LOOKUPS argument has subscripts. --- docs/whats_new.rst | 1 + .../python/python_expressions_builder.py | 20 ++++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/whats_new.rst b/docs/whats_new.rst index 48f22f10..9a46f396 100644 --- a/docs/whats_new.rst +++ b/docs/whats_new.rst @@ -14,6 +14,7 @@ Deprecations Bug fixes ~~~~~~~~~ +- Fix bug when a WITH LOOKUPS argument has subscripts. Documentation ~~~~~~~~~~~~~ diff --git a/pysd/builders/python/python_expressions_builder.py b/pysd/builders/python/python_expressions_builder.py index 7308a2a5..1c0b76fc 100644 --- a/pysd/builders/python/python_expressions_builder.py +++ b/pysd/builders/python/python_expressions_builder.py @@ -1769,11 +1769,21 @@ def build(self, arguments: dict) -> BuildAST: separator=",", threshold=len(self.lookups.y) ) - return BuildAST( - expression="np.interp(%(value)s, %(x)s, %(y)s)" % arguments, - calls=arguments["value"].calls, - subscripts=arguments["value"].subscripts, - order=0) + if arguments["value"].subscripts: + subs = arguments["value"].subscripts + expression = "np.interp(%(value)s, %(x)s, %(y)s)" % arguments + return BuildAST( + expression="xr.DataArray(%s, %s, %s)" % ( + expression, subs, list(subs)), + calls=arguments["value"].calls, + subscripts=subs, + order=0) + else: + return BuildAST( + expression="np.interp(%(value)s, %(x)s, %(y)s)" % arguments, + calls=arguments["value"].calls, + subscripts={}, + order=0) class ReferenceBuilder(StructureBuilder): From 47e6420e9b9c8392d1d98d286b2ed7d0d8860438 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez Date: Tue, 6 Sep 2022 22:28:16 +0200 Subject: [PATCH 03/43] Fix bug when a WITH LOOKUPS argument has subscripts. --- docs/whats_new.rst | 1 + .../python/python_expressions_builder.py | 20 ++++++++++++++----- .../pytest_integration_vensim_pathway.py | 4 ++++ tests/test-models | 2 +- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/whats_new.rst b/docs/whats_new.rst index 48f22f10..9a46f396 100644 --- a/docs/whats_new.rst +++ b/docs/whats_new.rst @@ -14,6 +14,7 @@ Deprecations Bug fixes ~~~~~~~~~ +- Fix bug when a WITH LOOKUPS argument has subscripts. Documentation ~~~~~~~~~~~~~ diff --git a/pysd/builders/python/python_expressions_builder.py b/pysd/builders/python/python_expressions_builder.py index 7308a2a5..1c0b76fc 100644 --- a/pysd/builders/python/python_expressions_builder.py +++ b/pysd/builders/python/python_expressions_builder.py @@ -1769,11 +1769,21 @@ def build(self, arguments: dict) -> BuildAST: separator=",", threshold=len(self.lookups.y) ) - return BuildAST( - expression="np.interp(%(value)s, %(x)s, %(y)s)" % arguments, - calls=arguments["value"].calls, - subscripts=arguments["value"].subscripts, - order=0) + if arguments["value"].subscripts: + subs = arguments["value"].subscripts + expression = "np.interp(%(value)s, %(x)s, %(y)s)" % arguments + return BuildAST( + expression="xr.DataArray(%s, %s, %s)" % ( + expression, subs, list(subs)), + calls=arguments["value"].calls, + subscripts=subs, + order=0) + else: + return BuildAST( + expression="np.interp(%(value)s, %(x)s, %(y)s)" % arguments, + calls=arguments["value"].calls, + subscripts={}, + order=0) class ReferenceBuilder(StructureBuilder): diff --git a/tests/pytest_integration/pytest_integration_vensim_pathway.py b/tests/pytest_integration/pytest_integration_vensim_pathway.py index ce8e28d3..2663c51d 100644 --- a/tests/pytest_integration/pytest_integration_vensim_pathway.py +++ b/tests/pytest_integration/pytest_integration_vensim_pathway.py @@ -546,6 +546,10 @@ "folder": "vector_select", "file": "test_vector_select.mdl" }, + "with_lookup": { + "folder": "with_lookup", + "file": "test_with_lookup.mdl" + }, "xidz_zidz": { "folder": "xidz_zidz", "file": "xidz_zidz.mdl" diff --git a/tests/test-models b/tests/test-models index 9a2f47c0..be07310e 160000 --- a/tests/test-models +++ b/tests/test-models @@ -1 +1 @@ -Subproject commit 9a2f47c042c11a09ebddb4ac142eb90467ed0f7d +Subproject commit be07310e8d45def1c7d2d973bac630274fd158b7 From 81c6c8ea379105abf9ab1edddf8eea051a47f83d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Fri, 2 Sep 2022 12:08:34 +0200 Subject: [PATCH 04/43] first implementation of netCDF storage --- pysd/py_backend/model.py | 115 ++++++++++++++++++++++++++++++++++++++- requirements.txt | 1 + 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index 60a35a0a..32f81bc0 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -8,11 +8,13 @@ import warnings import inspect import pickle +import time as t from typing import Union import numpy as np import xarray as xr import pandas as pd +import netCDF4 as nc from . import utils from .statefuls import DynamicStateful, Stateful @@ -1035,7 +1037,7 @@ def initialize(self): def run(self, params=None, return_columns=None, return_timestamps=None, initial_condition='original', final_time=None, time_step=None, saveper=None, reload=False, progress=False, flatten_output=True, - cache_output=True): + cache_output=True, output_file=None): """ Simulate the model's behavior over time. Return a pandas dataframe with timestamps as rows, @@ -1170,6 +1172,11 @@ def run(self, params=None, return_columns=None, return_timestamps=None, # need to clean cache to remove the values from active_initial self.clean_caches() + if output_file: + self._integrate_binary_output(capture_elements['step'], out_file=output_file) + print(f"Resulsts stored in {output_file}") + return None + res = self._integrate(capture_elements['step']) del self._dependencies["OUTPUTS"] @@ -1700,6 +1707,112 @@ def _integrate(self, capture_elements): del outputs["time"] return outputs + def _integrate_binary_output(self, capture_elements, out_file): + """ + Performs euler integration and stores the results in netCDF4 file. + + Parameters + ---------- + capture_elements: set + Which model elements to capture - uses pysafe names. + out_file: str or pathlib.Path + + Returns + ------- + nothing + + """ + + if self.progress: + # initialize progress bar + progressbar = utils.ProgressBar( + int((self.time.final_time()-self.time())/self.time.time_step()) + ) + else: + # when None is used the update will do nothing + progressbar = utils.ProgressBar(None) + + + with nc.Dataset(out_file, "w") as ds: + # generating global attributes + ds.description = "Results for simulation run on" \ + f"{t.ctime(t.time())} using PySD version {__version__}" + ds.model_file = self.py_model_file + ds.timestep = "{}".format(self.components.time_step()) + ds.initial_time = "{}".format(self.components.initial_time()) + ds.final_time = "{}".format(self.components.final_time()) + + # creating variables for all model dimensions + for dim_name,coords in self.subscripts.items(): + coords = np.array(coords) + # create dimension + ds.createDimension(dim_name, len(coords)) + # length of the longest string in the coords + max_str_len = len(max(coords, key=len)) + # create variable (TODO: check if the type could be defined otherwise) + var = ds.createVariable(dim_name, + f"S{max_str_len}", + (dim_name,)) + # assigning values to variable + var[:] = coords + + # creating the time dimension and variable + times = np.arange(self.components.initial_time(), + self.components.final_time(), + self.components.saveper() + ) + ds.createDimension("time", len(times)) + time = ds.createVariable("time", "f8", ("time",)) + time[:] = times + + step = 0 + while self.time.in_bounds(): + if self.time.in_return(): + for key in capture_elements: + comp = getattr(self.components, key) + comp_vals = comp() + + if step == 0: + dims = ("time",) + tuple(comp.subscripts) \ + if isinstance(comp_vals, (xr.DataArray, np.ndarray)) \ + else ("time",) + + ds.createVariable(key, "f8", dims) + + ds[key].units = self.doc.loc[ + self.doc["Py Name"] == key, + "Units"].values[0] or "Missing" + ds[key].description = self.doc.loc[ + self.doc["Py Name"] == key, + "Comment"].values[0] or "Missing" + + try: + if isinstance(comp_vals, xr.DataArray): + ds[key][step, :] = comp_vals.values + elif isinstance(comp_vals, np.ndarray): + ds[key][step, :] = comp_vals + else: + ds[key][step] = comp_vals + except: + print(key) + + + self._euler_step(self.time.time_step()) + self.time.update(self.time()+self.time.time_step()) + self.clean_caches() + progressbar.update() + step += 1 + + # TODO check this bit from the other method + # need to add one more time step, because we run only the state + # updates in the previous loop and thus may be one short. + #if self.time.in_return(): + # outputs.at[self.time.round()] = [getattr(self.components, key)() + # for key in capture_elements] + progressbar.finish() + + + def _add_run_elements(self, df, capture_elements): """ Adds constant elements to a dataframe. diff --git a/requirements.txt b/requirements.txt index e0947fec..d892ed41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ pandas parsimonious xarray +netCDF4 xlrd lxml regex From cfccb04aad0b8f4a426b7d46be63b01eea430963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Fri, 2 Sep 2022 23:46:58 +0200 Subject: [PATCH 05/43] using handlers --- pysd/py_backend/model.py | 333 ++++++++++++++++++++++++--------------- 1 file changed, 209 insertions(+), 124 deletions(-) diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index 32f81bc0..15c42986 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -1172,21 +1172,20 @@ def run(self, params=None, return_columns=None, return_timestamps=None, # need to clean cache to remove the values from active_initial self.clean_caches() - if output_file: - self._integrate_binary_output(capture_elements['step'], out_file=output_file) - print(f"Resulsts stored in {output_file}") - return None - - res = self._integrate(capture_elements['step']) + out_obj = self._integrate(capture_elements['step'], output_file) del self._dependencies["OUTPUTS"] - self._add_run_elements(res, capture_elements['run']) + out_obj.add_run_elements(self, capture_elements['run']) + self._remove_constant_cache() - return_df = utils.make_flat_df(res, return_addresses, flatten_output) + if output_file: + out_obj.handler.ds.close() + return None + + return utils.make_flat_df(out_obj.handler.ds, return_addresses, flatten_output) - return return_df def select_submodel(self, vars=[], modules=[], exogenous_components={}): """ @@ -1654,26 +1653,25 @@ def _euler_step(self, dt): """ self.state = self.state + self.ddt() * dt - def _integrate(self, capture_elements): + + def _integrate(self, capture_elements, out_file): """ - Performs euler integration. + Performs euler integration and stores the results in netCDF4 file. Parameters ---------- capture_elements: set Which model elements to capture - uses pysafe names. + out_file: str or pathlib.Path Returns ------- - outputs: pandas.DataFrame - Output capture_elements data. + nothing """ - # necessary to have always a non-xaray object for appending objects - # to the DataFrame time will always be a model element and not saved - # TODO: find a better way of saving outputs - capture_elements.add("time") - outputs = pd.DataFrame(columns=capture_elements) + # instantiating output object + output = ModelOutput(capture_elements, out_file) + output.initialize(self) if self.progress: # initialize progress bar @@ -1686,150 +1684,237 @@ def _integrate(self, capture_elements): while self.time.in_bounds(): if self.time.in_return(): - outputs.at[self.time.round()] = [ - getattr(self.components, key)() - for key in capture_elements] - self._euler_step(self.time.time_step()) - self.time.update(self.time()+self.time.time_step()) - self.clean_caches() - progressbar.update() + output.update(self) + + self._euler_step(self.time.time_step()) + self.time.update(self.time()+self.time.time_step()) + self.clean_caches() + progressbar.update() # need to add one more time step, because we run only the state # updates in the previous loop and thus may be one short. if self.time.in_return(): - outputs.at[self.time.round()] = [getattr(self.components, key)() - for key in capture_elements] + output.update(self) progressbar.finish() - # delete time column as it was created only for avoiding errors - # of appending data. See previous TODO. - del outputs["time"] - return outputs + output.postprocess() + return output + - def _integrate_binary_output(self, capture_elements, out_file): +class OutputHandler(object): + pass + +class DatasetHandler(OutputHandler): + + def __init__(self, out_file): + self.out_file = out_file + self.step = 0 + self.ds = None + + def initialize(self, model, capture_elements): """ - Performs euler integration and stores the results in netCDF4 file. + Creates a netCDF4 dataset and adds model dimensions and variables + present in the capture elements to it. Parameters ---------- capture_elements: set Which model elements to capture - uses pysafe names. out_file: str or pathlib.Path + Path to the file where the results will be written. + Returns + ------- + ds: netCDF4.Dataset + Initialized Dataset. + """ + self.ds = nc.Dataset(self.out_file, "w") + # generating global attributes + self.ds.description = "Results for simulation run on" \ + f"{t.ctime(t.time())} using PySD version {__version__}" + self.ds.model_file = model.py_model_file + self.ds.timestep = "{}".format(model.components.time_step()) + self.ds.initial_time = "{}".format(model.components.initial_time()) + self.ds.final_time = "{}".format(model.components.final_time()) + + # creating variables for all model dimensions + for dim_name,coords in model.subscripts.items(): + coords = np.array(coords) + # create dimension + self.ds.createDimension(dim_name, len(coords)) + # length of the longest string in the coords + max_str_len = len(max(coords, key=len)) + # create variable (TODO: check if the type could be defined + # otherwise) + var = self.ds.createVariable(dim_name, + f"S{max_str_len}", + (dim_name,)) + # assigning values to variable + var[:] = coords + + # creating the time dimension as unlimited + self.ds.createDimension("time", None) + + # creating variables in capture_elements + self.__create_ds_vars(model, capture_elements) + + def update(self, model, capture_elements): + + for key in capture_elements: + comp = getattr(model.components, key) + comp_vals = comp() + if isinstance(comp_vals, xr.DataArray): + self.ds[key][self.step, :] = comp_vals.values + elif isinstance(comp_vals, np.ndarray): + self.ds[key][self.step, :] = comp_vals + else: + self.ds[key][self.step] = comp_vals + + self.step += 1 + + def postprocess(self): + print(f"Resulsts stored in {self.out_file}") + + def add_run_elements(self, model, run_elements): + """ + Adds constant elements to a dataframe. + Parameters + ---------- + df: pandas.DataFrame + Dataframe to add elements. + run_elements: list + List of constant elements Returns ------- - nothing + None + """ + # creating variables in capture_elements + # TODO we are looping through all capture elements twice. This + # could be avoided + self.__create_ds_vars(model, run_elements) + + for key in run_elements: + comp = getattr(model.components, key) + comp_vals = comp() + for num,_ in enumerate(self.ds["time"][:]): + if isinstance(comp_vals, xr.DataArray): + self.ds[key][num, :] = comp_vals.values + elif isinstance(comp_vals, np.ndarray): + self.ds[key][num, :] = comp_vals + else: + self.ds[key][num] = comp_vals + def __create_ds_vars(self, model, capture_elements): """ + Create new variables in a netCDF4 Dataset from the capture_elements. - if self.progress: - # initialize progress bar - progressbar = utils.ProgressBar( - int((self.time.final_time()-self.time())/self.time.time_step()) - ) - else: - # when None is used the update will do nothing - progressbar = utils.ProgressBar(None) + Parameters + ---------- + ds: netCDF4.Dataset + Dataset in which to write the new variables. + capture_elements: set + Which model elements to capture - uses pysafe names. + Returns + ------- + ds: netCDF4.Dataset + Updated Dataset. - with nc.Dataset(out_file, "w") as ds: - # generating global attributes - ds.description = "Results for simulation run on" \ - f"{t.ctime(t.time())} using PySD version {__version__}" - ds.model_file = self.py_model_file - ds.timestep = "{}".format(self.components.time_step()) - ds.initial_time = "{}".format(self.components.initial_time()) - ds.final_time = "{}".format(self.components.final_time()) - - # creating variables for all model dimensions - for dim_name,coords in self.subscripts.items(): - coords = np.array(coords) - # create dimension - ds.createDimension(dim_name, len(coords)) - # length of the longest string in the coords - max_str_len = len(max(coords, key=len)) - # create variable (TODO: check if the type could be defined otherwise) - var = ds.createVariable(dim_name, - f"S{max_str_len}", - (dim_name,)) - # assigning values to variable - var[:] = coords - - # creating the time dimension and variable - times = np.arange(self.components.initial_time(), - self.components.final_time(), - self.components.saveper() - ) - ds.createDimension("time", len(times)) - time = ds.createVariable("time", "f8", ("time",)) - time[:] = times - - step = 0 - while self.time.in_bounds(): - if self.time.in_return(): - for key in capture_elements: - comp = getattr(self.components, key) - comp_vals = comp() - - if step == 0: - dims = ("time",) + tuple(comp.subscripts) \ - if isinstance(comp_vals, (xr.DataArray, np.ndarray)) \ - else ("time",) - - ds.createVariable(key, "f8", dims) - - ds[key].units = self.doc.loc[ - self.doc["Py Name"] == key, - "Units"].values[0] or "Missing" - ds[key].description = self.doc.loc[ - self.doc["Py Name"] == key, - "Comment"].values[0] or "Missing" - - try: - if isinstance(comp_vals, xr.DataArray): - ds[key][step, :] = comp_vals.values - elif isinstance(comp_vals, np.ndarray): - ds[key][step, :] = comp_vals - else: - ds[key][step] = comp_vals - except: - print(key) + """ + for key in capture_elements: + comp = getattr(model.components, key) + comp_vals = comp() - self._euler_step(self.time.time_step()) - self.time.update(self.time()+self.time.time_step()) - self.clean_caches() - progressbar.update() - step += 1 - - # TODO check this bit from the other method - # need to add one more time step, because we run only the state - # updates in the previous loop and thus may be one short. - #if self.time.in_return(): - # outputs.at[self.time.round()] = [getattr(self.components, key)() - # for key in capture_elements] - progressbar.finish() + dims = ("time",) + tuple(comp.subscripts) \ + if isinstance(comp_vals, (xr.DataArray, np.ndarray)) \ + else ("time",) + self.ds.createVariable(key, "f8", dims) + # adding units and description as metadata for each var + self.ds[key].units = model.doc.loc[ + model.doc["Py Name"] == key, + "Units"].values[0] or "Missing" + self.ds[key].description = model.doc.loc[ + model.doc["Py Name"] == key, + "Comment"].values[0] or "Missing" - def _add_run_elements(self, df, capture_elements): + + +class DataFrameHandler(OutputHandler): + + def __init__(self): + self.ds = None + + def initialize(self, model, capture_elements): """ - Adds constant elements to a dataframe. + Creates a pandas DataFrame and adds model variables as columns Parameters ---------- + capture_elements: set + Which model elements to capture - uses pysafe names. + + Returns + ------- df: pandas.DataFrame - Dataframe to add elements. + Initialized Dataset. + + """ + self.ds = pd.DataFrame(columns=capture_elements) + + def update(self, model, capture_elements): + + self.ds.at[model.time.round()] = [ + getattr(model.components, key)() + for key in capture_elements] + + def postprocess(self): + # delete time column as it was created only for avoiding errors + # of appending data. See previous TODO. + del self.ds["time"] + + def add_run_elements(self, model, capture_elements): + """ + Adds constant elements to a dataframe. + Parameters + ---------- + df: pandas.DataFrame + Dataframe to add elements. capture_elements: list List of constant elements - Returns ------- None - """ - nt = len(df.index.values) + nx = len(self.ds.index) for element in capture_elements: - df[element] = [getattr(self.components, element)()] * nt + self.ds[element] = [getattr(model.components, element)()] * nx + +class ModelOutput(): + + def __init__(self, capture_elements, out_file=None): + + if out_file: + self.handler = DatasetHandler(out_file) + else: + self.handler = DataFrameHandler() + + capture_elements.add("time") + self.capture_elements = capture_elements + + def initialize(self, model): + self.handler.initialize(model, self.capture_elements) + + def update(self, model): + self.handler.update(model, self.capture_elements) + + def postprocess(self): + self.handler.postprocess() + + def add_run_elements(self, model, run_elements): + self.handler.add_run_elements(model, run_elements) + From 615d8d18730f5ed67b6eeb18b3e14357737803a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Sat, 3 Sep 2022 14:25:34 +0200 Subject: [PATCH 06/43] improving postprocess and docs --- pysd/py_backend/model.py | 237 ++++++++++++++++++++++++++++----------- 1 file changed, 172 insertions(+), 65 deletions(-) diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index 15c42986..24de08d5 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -1160,6 +1160,7 @@ def run(self, params=None, return_columns=None, return_timestamps=None, self._dependencies["OUTPUTS"] = { element: 1 for element in capture_elements["step"] } + if cache_output: # udate the cache type taking into account the outputs self._assign_cache_type() @@ -1172,19 +1173,19 @@ def run(self, params=None, return_columns=None, return_timestamps=None, # need to clean cache to remove the values from active_initial self.clean_caches() - out_obj = self._integrate(capture_elements['step'], output_file) + # instantiating output object + output = ModelOutput(self, capture_elements['step'], output_file) + + self._integrate(output) del self._dependencies["OUTPUTS"] - out_obj.add_run_elements(self, capture_elements['run']) + output.add_run_elements(self, capture_elements['run']) self._remove_constant_cache() - if output_file: - out_obj.handler.ds.close() - return None - - return utils.make_flat_df(out_obj.handler.ds, return_addresses, flatten_output) + return output.postprocess(return_addresses=return_addresses, + flatten=flatten_output) def select_submodel(self, vars=[], modules=[], exogenous_components={}): @@ -1654,24 +1655,19 @@ def _euler_step(self, dt): self.state = self.state + self.ddt() * dt - def _integrate(self, capture_elements, out_file): + def _integrate(self, out_obj): """ - Performs euler integration and stores the results in netCDF4 file. + Performs euler integration and writes results to the out_obj. Parameters ---------- - capture_elements: set - Which model elements to capture - uses pysafe names. - out_file: str or pathlib.Path + out_obj: pysd.ModelOutput Returns ------- - nothing + None """ - # instantiating output object - output = ModelOutput(capture_elements, out_file) - output.initialize(self) if self.progress: # initialize progress bar @@ -1682,30 +1678,44 @@ def _integrate(self, capture_elements, out_file): # when None is used the update will do nothing progressbar = utils.ProgressBar(None) + # performs the time stepping while self.time.in_bounds(): if self.time.in_return(): - output.update(self) + out_obj.update(self) - self._euler_step(self.time.time_step()) - self.time.update(self.time()+self.time.time_step()) - self.clean_caches() - progressbar.update() + self._euler_step(self.time.time_step()) + self.time.update(self.time()+self.time.time_step()) + self.clean_caches() + progressbar.update() # need to add one more time step, because we run only the state # updates in the previous loop and thus may be one short. if self.time.in_return(): - output.update(self) + out_obj.update(self) progressbar.finish() - output.postprocess() - return output +class OutputHandler(object): + """ + Interface for the different output handlers. + """ + + def initialize(self): + pass + + def update(self): + pass + def postprocess(self): + pass -class OutputHandler(object): - pass + def add_run_elements(self): + pass class DatasetHandler(OutputHandler): + """ + Manages simulation results stored as netCDF4 Dataset. + """ def __init__(self, out_file): self.out_file = out_file @@ -1714,23 +1724,23 @@ def __init__(self, out_file): def initialize(self, model, capture_elements): """ - Creates a netCDF4 dataset and adds model dimensions and variables + Creates a netCDF4 Dataset and adds model dimensions and variables present in the capture elements to it. Parameters ---------- + model: pysd.Model + PySD Model object capture_elements: set Which model elements to capture - uses pysafe names. - out_file: str or pathlib.Path - Path to the file where the results will be written. + Returns ------- - ds: netCDF4.Dataset - Initialized Dataset. + None """ self.ds = nc.Dataset(self.out_file, "w") - # generating global attributes + # defining global attributes self.ds.description = "Results for simulation run on" \ f"{t.ctime(t.time())} using PySD version {__version__}" self.ds.model_file = model.py_model_file @@ -1745,8 +1755,9 @@ def initialize(self, model, capture_elements): self.ds.createDimension(dim_name, len(coords)) # length of the longest string in the coords max_str_len = len(max(coords, key=len)) - # create variable (TODO: check if the type could be defined - # otherwise) + + # create variable + # TODO: check if the type could be defined otherwise) var = self.ds.createVariable(dim_name, f"S{max_str_len}", (dim_name,)) @@ -1759,7 +1770,23 @@ def initialize(self, model, capture_elements): # creating variables in capture_elements self.__create_ds_vars(model, capture_elements) + return None + def update(self, model, capture_elements): + """ + Writes values of variables in capture_elements in the netCDF4 Dataset. + + Parameters + ---------- + model: pysd.Model + PySD Model object + capture_elements: set + Which model elements to capture - uses pysafe names. + + Returns + ------- + None + """ for key in capture_elements: comp = getattr(model.components, key) @@ -1773,16 +1800,29 @@ def update(self, model, capture_elements): self.step += 1 - def postprocess(self): - print(f"Resulsts stored in {self.out_file}") + return None + + def postprocess(self, **kwargs): + """ + Closes netCDF4 Dataset. + """ + + self.ds.close() + + if kwargs.get("flatten"): + warnings.warn("DataArrays stored in netCDF4 will not be flattened") + + print(f"Results stored in {self.out_file}") + + return None def add_run_elements(self, model, run_elements): """ - Adds constant elements to a dataframe. + Adds constant elements to netCDF4 Dataset. Parameters ---------- - df: pandas.DataFrame - Dataframe to add elements. + model: pysd.Model + PySD Model object run_elements: list List of constant elements Returns @@ -1797,13 +1837,23 @@ def add_run_elements(self, model, run_elements): for key in run_elements: comp = getattr(model.components, key) comp_vals = comp() - for num,_ in enumerate(self.ds["time"][:]): - if isinstance(comp_vals, xr.DataArray): - self.ds[key][num, :] = comp_vals.values - elif isinstance(comp_vals, np.ndarray): - self.ds[key][num, :] = comp_vals - else: - self.ds[key][num] = comp_vals + try: + for num,_ in enumerate(self.ds["time"][:]): + if isinstance(comp_vals, xr.DataArray): + self.ds[key][num, :] = comp_vals.values + elif isinstance(comp_vals, np.ndarray): + if comp_vals.size == 1: + self.ds[key][num] = comp_vals + else: + self.ds[key][num, :] = comp_vals + else: + self.ds[key][num] = comp_vals + except Exception as e: + warnings.warn(f"The dimensions of {key} in the results " + "do not match the declared dimensions for this " + "variable. The resulting values will not be " + "included in the results file.") + return None def __create_ds_vars(self, model, capture_elements): """ @@ -1811,15 +1861,14 @@ def __create_ds_vars(self, model, capture_elements): Parameters ---------- - ds: netCDF4.Dataset - Dataset in which to write the new variables. + model: pysd.Model + PySD Model object capture_elements: set Which model elements to capture - uses pysafe names. Returns ------- - ds: netCDF4.Dataset - Updated Dataset. + None """ @@ -1827,9 +1876,13 @@ def __create_ds_vars(self, model, capture_elements): comp = getattr(model.components, key) comp_vals = comp() - dims = ("time",) + tuple(comp.subscripts) \ - if isinstance(comp_vals, (xr.DataArray, np.ndarray)) \ - else ("time",) + if isinstance(comp_vals, (xr.DataArray, np.ndarray)): + if comp.subscripts: + dims = ("time",) + tuple(comp.subscripts) + else: + dims = ("time",) + else: + dims = ("time",) self.ds.createVariable(key, "f8", dims) @@ -1841,10 +1894,12 @@ def __create_ds_vars(self, model, capture_elements): model.doc["Py Name"] == key, "Comment"].values[0] or "Missing" - + return None class DataFrameHandler(OutputHandler): - + """ + Manages simulation results stored as pandas DataFrame. + """ def __init__(self): self.ds = None @@ -1854,38 +1909,72 @@ def initialize(self, model, capture_elements): Parameters ---------- + model: pysd.Model + PySD Model object capture_elements: set Which model elements to capture - uses pysafe names. Returns ------- - df: pandas.DataFrame - Initialized Dataset. + None """ self.ds = pd.DataFrame(columns=capture_elements) + return None def update(self, model, capture_elements): + """ + Add a row to the results pandas DataFrame with the values of the + variables listed in capture_elements. + + Parameters + ---------- + model: pysd.Model + PySD Model object + capture_elements: set + Which model elements to capture - uses pysafe names. + + Returns + ------- + None + """ self.ds.at[model.time.round()] = [ getattr(model.components, key)() for key in capture_elements] - def postprocess(self): + return None + + def postprocess(self, **kwargs): + """ + Delete time column from the pandas DataFrame and flatten xarrays if + required. + + Returns + ------- + ds: pandas.DataFrame + Simulation results stored as a pandas DataFrame. + """ # delete time column as it was created only for avoiding errors # of appending data. See previous TODO. del self.ds["time"] + return utils.make_flat_df(self.ds, + kwargs["return_addresses"], + kwargs["flatten"]) + def add_run_elements(self, model, capture_elements): """ Adds constant elements to a dataframe. + Parameters ---------- - df: pandas.DataFrame - Dataframe to add elements. - capture_elements: list - List of constant elements + model: pysd.Model + PySD Model object + capture_elements: set + Which model elements to capture - uses pysafe names. + Returns ------- None @@ -1894,9 +1983,24 @@ def add_run_elements(self, model, capture_elements): for element in capture_elements: self.ds[element] = [getattr(model.components, element)()] * nx + return None + class ModelOutput(): + """ + Handles different types of outputs by dispatchinging the tasks to adequate + object handlers. - def __init__(self, capture_elements, out_file=None): + Parameters + ---------- + model: pysd.Model + PySD Model object + capture_elements: set + Which model elements to capture - uses pysafe names. + out_file: str or pathlib.Path + Path to the file where the results will be written. + """ + + def __init__(self, model, capture_elements, out_file=None): if out_file: self.handler = DatasetHandler(out_file) @@ -1906,14 +2010,17 @@ def __init__(self, capture_elements, out_file=None): capture_elements.add("time") self.capture_elements = capture_elements + self.initialize(model) + def initialize(self, model): self.handler.initialize(model, self.capture_elements) def update(self, model): self.handler.update(model, self.capture_elements) - def postprocess(self): - self.handler.postprocess() + def postprocess(self, **kwargs): + processed = self.handler.postprocess(**kwargs) + return processed def add_run_elements(self, model, run_elements): self.handler.add_run_elements(model, run_elements) From 8f59ff68cf53f5a3f452e90a1d7433272e05f8fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Sat, 3 Sep 2022 15:07:02 +0200 Subject: [PATCH 07/43] handling supported output files --- pysd/py_backend/model.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index 24de08d5..3f811430 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -5,10 +5,12 @@ allows integrating a model or a Macro expression (set of functions in a separate file). """ +from io import UnsupportedOperation import warnings import inspect import pickle import time as t +from pathlib import Path from typing import Union import numpy as np @@ -1108,6 +1110,11 @@ def run(self, params=None, return_columns=None, return_timestamps=None, recommended to activate this feature, if time step << saveper it is recommended to deactivate it. Default is True. + output_file: str, pathlib.Path or None (optional) + Path of the file in which to save simulation results. + For now, only netCDF4 files (.nc) are supported. + + Examples -------- >>> model.run(params={'exogenous_constant': 42}) @@ -1165,6 +1172,21 @@ def run(self, params=None, return_columns=None, return_timestamps=None, # udate the cache type taking into account the outputs self._assign_cache_type() + # check validitty of output_file + if output_file: + if isinstance(output_file, str): + output_file = Path(output_file) + ext = output_file.suffix + elif isinstance(output_file, Path): + ext = output_file.suffix + else: + raise TypeError( + "Paths must be strings or pathlib Path objects.") + + if ext not in ModelOutput.valid_output_files: + raise UnsupportedOperation( + f"Unsupported output file format {ext}") + # add constant cache to thosa variable that are constants self._add_constant_cache() @@ -1999,11 +2021,16 @@ class ModelOutput(): out_file: str or pathlib.Path Path to the file where the results will be written. """ + valid_output_files = [".nc"] def __init__(self, model, capture_elements, out_file=None): if out_file: - self.handler = DatasetHandler(out_file) + if out_file.suffix == ".nc": + self.handler = DatasetHandler(out_file) + else: + raise UnsupportedOperation( + f"Unsupported output file format {out_file.suffix}") else: self.handler = DataFrameHandler() From 6d658ecfb07189c844952758d17e863d1895bf11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Sat, 3 Sep 2022 20:39:02 +0200 Subject: [PATCH 08/43] fix failing test --- pysd/py_backend/model.py | 5 ++--- tests/pytest_pysd/pytest_pysd.py | 38 ++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index 3f811430..1f8e6a6f 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -5,7 +5,6 @@ allows integrating a model or a Macro expression (set of functions in a separate file). """ -from io import UnsupportedOperation import warnings import inspect import pickle @@ -1184,7 +1183,7 @@ def run(self, params=None, return_columns=None, return_timestamps=None, "Paths must be strings or pathlib Path objects.") if ext not in ModelOutput.valid_output_files: - raise UnsupportedOperation( + raise ValueError( f"Unsupported output file format {ext}") # add constant cache to thosa variable that are constants @@ -2029,7 +2028,7 @@ def __init__(self, model, capture_elements, out_file=None): if out_file.suffix == ".nc": self.handler = DatasetHandler(out_file) else: - raise UnsupportedOperation( + raise ValueError( f"Unsupported output file format {out_file.suffix}") else: self.handler = DataFrameHandler() diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index 079723a1..54f81cde 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -5,6 +5,7 @@ import pandas as pd import numpy as np import xarray as xr +import netCDF4 as nc from pysd.tools.benchmarking import assert_frames_close @@ -207,6 +208,20 @@ def test_run_return_columns_pysafe_names(self): result = model.run(return_columns=return_columns) assert set(result.columns) == set(return_columns) + def test_run_output_file(self): + + model = pysd.read_vensim(test_model) + model.progress = False + + error_message = "Paths must be strings or pathlib Path objects." + with pytest.raises(TypeError, match=error_message): + model.run(output_file=1234) + + error_message = "Unsupported output file format .txt" + with pytest.raises(ValueError, match=error_message): + model.run(output_file="file.txt") + + def test_initial_conditions_invalid(self): model = pysd.read_vensim(test_model) error_message = r"Invalid initial conditions\. "\ @@ -1178,16 +1193,34 @@ def test_get_series_data(self): data = model3.get_series_data('_ext_data_data_backward') assert data.equals(data_exp) - def test__integrate(self): + def test__integrate(self, shared_tmpdir): + from pysd.py_backend.model import ModelOutput # Todo: think through a stronger test here... model = pysd.read_vensim(test_model) model.progress = False model.time.add_return_timestamps(list(range(0, 5, 2))) - res = model._integrate(capture_elements={'teacup_temperature'}) + capture_elements={'teacup_temperature'} + + out = ModelOutput(model, capture_elements, None) + model._integrate(out) + res = out.handler.ds assert isinstance(res, pd.DataFrame) assert 'teacup_temperature' in res assert all(res.index.values == list(range(0, 5, 2))) + model = pysd.read_vensim(test_model) + model.progress = False + model.time.add_return_timestamps(list(range(0, 5, 2))) + out = ModelOutput(model, + capture_elements, + shared_tmpdir.joinpath("output.nc")) + model._integrate(out) + res = out.handler.ds + assert isinstance(res, nc.Dataset) + assert 'teacup_temperature' in res.variables + assert np.array_equal(res["time"][:].data, np.arange(0, 5, 2)) + res.close() + def test_default_returns_with_construction_functions(self): """ If the run function is called with no arguments, should still be able @@ -1646,3 +1679,4 @@ def test_run_export_import_initial(self): Path('initial7.pic').unlink() assert_frames_close(stocks2, stocks) + From 6f5cf66c8c0ccc7ca76be3da76d8e8b9dc5e6865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Sat, 3 Sep 2022 22:51:38 +0200 Subject: [PATCH 09/43] add h5py in requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index d892ed41..1661adbf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ pandas parsimonious xarray +h5py netCDF4 xlrd lxml From dc93b8b416cdad6bee17962e9d0d40b82f7f3391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Sat, 3 Sep 2022 23:37:28 +0200 Subject: [PATCH 10/43] fix pep8? --- pysd/py_backend/model.py | 45 ++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index 1f8e6a6f..46655b83 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -712,12 +712,11 @@ def set_components(self, params, new=False): >>> br = pandas.Series(index=range(30), values=np.sin(range(30)) >>> model.set_components({'birth_rate': br}) - """ - # TODO: allow the params argument to take a pandas dataframe, where + # TODO: allow the params argument to take a pandas dataframe, where # column names are variable names. However some variables may be # constant or have no values for some index. This should be processed. - # TODO: make this compatible with loading outputs from other files + # TODO: make this compatible with loading outputs from other files for key, value in params.items(): func_name = utils.get_key_and_value_by_insensitive_key_or_value( @@ -1208,7 +1207,6 @@ def run(self, params=None, return_columns=None, return_timestamps=None, return output.postprocess(return_addresses=return_addresses, flatten=flatten_output) - def select_submodel(self, vars=[], modules=[], exogenous_components={}): """ Select a submodel from the original model. After selecting a submodel @@ -1716,23 +1714,25 @@ def _integrate(self, out_obj): progressbar.finish() -class OutputHandler(object): + +class OutputHandler(): """ Interface for the different output handlers. """ - def initialize(self): + def initialize(self, model, capture_elements): pass - def update(self): + def update(self, model, capture_elements): pass - def postprocess(self): + def postprocess(self, **kwargs): pass - def add_run_elements(self): + def add_run_elements(self, model, capture_elements): pass + class DatasetHandler(OutputHandler): """ Manages simulation results stored as netCDF4 Dataset. @@ -1791,8 +1791,6 @@ def initialize(self, model, capture_elements): # creating variables in capture_elements self.__create_ds_vars(model, capture_elements) - return None - def update(self, model, capture_elements): """ Writes values of variables in capture_elements in the netCDF4 Dataset. @@ -1808,7 +1806,6 @@ def update(self, model, capture_elements): ------- None """ - for key in capture_elements: comp = getattr(model.components, key) comp_vals = comp() @@ -1821,13 +1818,10 @@ def update(self, model, capture_elements): self.step += 1 - return None - def postprocess(self, **kwargs): """ Closes netCDF4 Dataset. """ - self.ds.close() if kwargs.get("flatten"): @@ -1835,16 +1829,14 @@ def postprocess(self, **kwargs): print(f"Results stored in {self.out_file}") - return None - - def add_run_elements(self, model, run_elements): + def add_run_elements(self, model, capture_elements): """ Adds constant elements to netCDF4 Dataset. Parameters ---------- model: pysd.Model PySD Model object - run_elements: list + capture_elements: list List of constant elements Returns ------- @@ -1853,9 +1845,9 @@ def add_run_elements(self, model, run_elements): # creating variables in capture_elements # TODO we are looping through all capture elements twice. This # could be avoided - self.__create_ds_vars(model, run_elements) + self.__create_ds_vars(model, capture_elements) - for key in run_elements: + for key in capture_elements: comp = getattr(model.components, key) comp_vals = comp() try: @@ -1869,12 +1861,11 @@ def add_run_elements(self, model, run_elements): self.ds[key][num, :] = comp_vals else: self.ds[key][num] = comp_vals - except Exception as e: + except ValueError: warnings.warn(f"The dimensions of {key} in the results " "do not match the declared dimensions for this " "variable. The resulting values will not be " "included in the results file.") - return None def __create_ds_vars(self, model, capture_elements): """ @@ -1915,7 +1906,6 @@ def __create_ds_vars(self, model, capture_elements): model.doc["Py Name"] == key, "Comment"].values[0] or "Missing" - return None class DataFrameHandler(OutputHandler): """ @@ -1965,8 +1955,6 @@ def update(self, model, capture_elements): getattr(model.components, key)() for key in capture_elements] - return None - def postprocess(self, **kwargs): """ Delete time column from the pandas DataFrame and flatten xarrays if @@ -2004,7 +1992,6 @@ def add_run_elements(self, model, capture_elements): for element in capture_elements: self.ds[element] = [getattr(model.components, element)()] * nx - return None class ModelOutput(): """ @@ -2045,9 +2032,7 @@ def update(self, model): self.handler.update(model, self.capture_elements) def postprocess(self, **kwargs): - processed = self.handler.postprocess(**kwargs) - return processed + return self.handler.postprocess(**kwargs) def add_run_elements(self, model, run_elements): self.handler.add_run_elements(model, run_elements) - From c85a8bc397953e5ef0d2a34c85d4700832f76ccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Mon, 5 Sep 2022 09:51:05 +0200 Subject: [PATCH 11/43] move output classes to separate file --- pysd/py_backend/model.py | 332 +-------------------------- pysd/py_backend/output.py | 381 +++++++++++++++++++++++++++++++ tests/pytest_pysd/pytest_pysd.py | 4 +- 3 files changed, 387 insertions(+), 330 deletions(-) create mode 100644 pysd/py_backend/output.py diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index 46655b83..6ee741f1 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -8,14 +8,14 @@ import warnings import inspect import pickle -import time as t from pathlib import Path from typing import Union import numpy as np import xarray as xr import pandas as pd -import netCDF4 as nc + +from pysd._version import __version__ from . import utils from .statefuls import DynamicStateful, Stateful @@ -24,8 +24,7 @@ from .data import TabData from .lookups import HardcodedLookups from .components import Components, Time - -from pysd._version import __version__ +from .output import ModelOutput class Macro(DynamicStateful): @@ -1025,6 +1024,7 @@ def __init__(self, py_model_file, data_files, initialize, missing_values): self.time.set_control_vars(**self.components._control_vars) self.data_files = data_files self.missing_values = missing_values + self.progress = None if initialize: self.initialize() @@ -1673,7 +1673,6 @@ def _euler_step(self, dt): """ self.state = self.state + self.ddt() * dt - def _integrate(self, out_obj): """ Performs euler integration and writes results to the out_obj. @@ -1713,326 +1712,3 @@ def _integrate(self, out_obj): out_obj.update(self) progressbar.finish() - - -class OutputHandler(): - """ - Interface for the different output handlers. - """ - - def initialize(self, model, capture_elements): - pass - - def update(self, model, capture_elements): - pass - - def postprocess(self, **kwargs): - pass - - def add_run_elements(self, model, capture_elements): - pass - - -class DatasetHandler(OutputHandler): - """ - Manages simulation results stored as netCDF4 Dataset. - """ - - def __init__(self, out_file): - self.out_file = out_file - self.step = 0 - self.ds = None - - def initialize(self, model, capture_elements): - """ - Creates a netCDF4 Dataset and adds model dimensions and variables - present in the capture elements to it. - - Parameters - ---------- - model: pysd.Model - PySD Model object - capture_elements: set - Which model elements to capture - uses pysafe names. - - Returns - ------- - None - - """ - self.ds = nc.Dataset(self.out_file, "w") - # defining global attributes - self.ds.description = "Results for simulation run on" \ - f"{t.ctime(t.time())} using PySD version {__version__}" - self.ds.model_file = model.py_model_file - self.ds.timestep = "{}".format(model.components.time_step()) - self.ds.initial_time = "{}".format(model.components.initial_time()) - self.ds.final_time = "{}".format(model.components.final_time()) - - # creating variables for all model dimensions - for dim_name,coords in model.subscripts.items(): - coords = np.array(coords) - # create dimension - self.ds.createDimension(dim_name, len(coords)) - # length of the longest string in the coords - max_str_len = len(max(coords, key=len)) - - # create variable - # TODO: check if the type could be defined otherwise) - var = self.ds.createVariable(dim_name, - f"S{max_str_len}", - (dim_name,)) - # assigning values to variable - var[:] = coords - - # creating the time dimension as unlimited - self.ds.createDimension("time", None) - - # creating variables in capture_elements - self.__create_ds_vars(model, capture_elements) - - def update(self, model, capture_elements): - """ - Writes values of variables in capture_elements in the netCDF4 Dataset. - - Parameters - ---------- - model: pysd.Model - PySD Model object - capture_elements: set - Which model elements to capture - uses pysafe names. - - Returns - ------- - None - """ - for key in capture_elements: - comp = getattr(model.components, key) - comp_vals = comp() - if isinstance(comp_vals, xr.DataArray): - self.ds[key][self.step, :] = comp_vals.values - elif isinstance(comp_vals, np.ndarray): - self.ds[key][self.step, :] = comp_vals - else: - self.ds[key][self.step] = comp_vals - - self.step += 1 - - def postprocess(self, **kwargs): - """ - Closes netCDF4 Dataset. - """ - self.ds.close() - - if kwargs.get("flatten"): - warnings.warn("DataArrays stored in netCDF4 will not be flattened") - - print(f"Results stored in {self.out_file}") - - def add_run_elements(self, model, capture_elements): - """ - Adds constant elements to netCDF4 Dataset. - Parameters - ---------- - model: pysd.Model - PySD Model object - capture_elements: list - List of constant elements - Returns - ------- - None - """ - # creating variables in capture_elements - # TODO we are looping through all capture elements twice. This - # could be avoided - self.__create_ds_vars(model, capture_elements) - - for key in capture_elements: - comp = getattr(model.components, key) - comp_vals = comp() - try: - for num,_ in enumerate(self.ds["time"][:]): - if isinstance(comp_vals, xr.DataArray): - self.ds[key][num, :] = comp_vals.values - elif isinstance(comp_vals, np.ndarray): - if comp_vals.size == 1: - self.ds[key][num] = comp_vals - else: - self.ds[key][num, :] = comp_vals - else: - self.ds[key][num] = comp_vals - except ValueError: - warnings.warn(f"The dimensions of {key} in the results " - "do not match the declared dimensions for this " - "variable. The resulting values will not be " - "included in the results file.") - - def __create_ds_vars(self, model, capture_elements): - """ - Create new variables in a netCDF4 Dataset from the capture_elements. - - Parameters - ---------- - model: pysd.Model - PySD Model object - capture_elements: set - Which model elements to capture - uses pysafe names. - - Returns - ------- - None - - """ - - for key in capture_elements: - comp = getattr(model.components, key) - comp_vals = comp() - - if isinstance(comp_vals, (xr.DataArray, np.ndarray)): - if comp.subscripts: - dims = ("time",) + tuple(comp.subscripts) - else: - dims = ("time",) - else: - dims = ("time",) - - self.ds.createVariable(key, "f8", dims) - - # adding units and description as metadata for each var - self.ds[key].units = model.doc.loc[ - model.doc["Py Name"] == key, - "Units"].values[0] or "Missing" - self.ds[key].description = model.doc.loc[ - model.doc["Py Name"] == key, - "Comment"].values[0] or "Missing" - - -class DataFrameHandler(OutputHandler): - """ - Manages simulation results stored as pandas DataFrame. - """ - def __init__(self): - self.ds = None - - def initialize(self, model, capture_elements): - """ - Creates a pandas DataFrame and adds model variables as columns - - Parameters - ---------- - model: pysd.Model - PySD Model object - capture_elements: set - Which model elements to capture - uses pysafe names. - - Returns - ------- - None - - """ - self.ds = pd.DataFrame(columns=capture_elements) - - return None - - def update(self, model, capture_elements): - """ - Add a row to the results pandas DataFrame with the values of the - variables listed in capture_elements. - - Parameters - ---------- - model: pysd.Model - PySD Model object - capture_elements: set - Which model elements to capture - uses pysafe names. - - Returns - ------- - None - """ - - self.ds.at[model.time.round()] = [ - getattr(model.components, key)() - for key in capture_elements] - - def postprocess(self, **kwargs): - """ - Delete time column from the pandas DataFrame and flatten xarrays if - required. - - Returns - ------- - ds: pandas.DataFrame - Simulation results stored as a pandas DataFrame. - """ - # delete time column as it was created only for avoiding errors - # of appending data. See previous TODO. - del self.ds["time"] - - return utils.make_flat_df(self.ds, - kwargs["return_addresses"], - kwargs["flatten"]) - - def add_run_elements(self, model, capture_elements): - """ - Adds constant elements to a dataframe. - - Parameters - ---------- - model: pysd.Model - PySD Model object - capture_elements: set - Which model elements to capture - uses pysafe names. - - Returns - ------- - None - """ - nx = len(self.ds.index) - for element in capture_elements: - self.ds[element] = [getattr(model.components, element)()] * nx - - -class ModelOutput(): - """ - Handles different types of outputs by dispatchinging the tasks to adequate - object handlers. - - Parameters - ---------- - model: pysd.Model - PySD Model object - capture_elements: set - Which model elements to capture - uses pysafe names. - out_file: str or pathlib.Path - Path to the file where the results will be written. - """ - valid_output_files = [".nc"] - - def __init__(self, model, capture_elements, out_file=None): - - if out_file: - if out_file.suffix == ".nc": - self.handler = DatasetHandler(out_file) - else: - raise ValueError( - f"Unsupported output file format {out_file.suffix}") - else: - self.handler = DataFrameHandler() - - capture_elements.add("time") - self.capture_elements = capture_elements - - self.initialize(model) - - def initialize(self, model): - self.handler.initialize(model, self.capture_elements) - - def update(self, model): - self.handler.update(model, self.capture_elements) - - def postprocess(self, **kwargs): - return self.handler.postprocess(**kwargs) - - def add_run_elements(self, model, run_elements): - self.handler.add_run_elements(model, run_elements) diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py new file mode 100644 index 00000000..f5aa55dd --- /dev/null +++ b/pysd/py_backend/output.py @@ -0,0 +1,381 @@ +""" +ModelOutput class is used to build different output objects based on +user input. For now, available output types are pandas DataFrame or +netCDF4 Dataset. +The OutputHandlerInterface class is an interface for the creation of handlers +for other output object types. +""" +import abc +import warnings +import time as t + +import numpy as np +import xarray as xr +import pandas as pd +import netCDF4 as nc + +from pysd._version import __version__ + +from . import utils + + +class ModelOutput(): + """ + Handles different types of outputs by dispatchinging the tasks to adequate + object handlers. + + Parameters + ---------- + model: pysd.Model + PySD Model object + capture_elements: set + Which model elements to capture - uses pysafe names. + out_file: str or pathlib.Path + Path to the file where the results will be written. + """ + valid_output_files = [".nc"] + + def __init__(self, model, capture_elements, out_file=None): + + self.handler = self.__handle(out_file) + + capture_elements.add("time") + self.capture_elements = capture_elements + + self.initialize(model) + + def __handle(self, out_file): + # TODO improve the handler to avoid if then else statements + if out_file: + if out_file.suffix == ".nc": + return DatasetHandler(out_file) + raise ValueError( + f"Unsupported output file format {out_file.suffix}") + return DataFrameHandler() + + def initialize(self, model): + """ Delegating the creation of the results object and its elements to + the appropriate handler.""" + self.handler.initialize(model, self.capture_elements) + + def update(self, model): + """ Delegating the update of the results object and its elements to the + appropriate handler.""" + self.handler.update(model, self.capture_elements) + + def postprocess(self, **kwargs): + """ Delegating the postprocessing of the results object to the + appropriate handler.""" + return self.handler.postprocess(**kwargs) + + def add_run_elements(self, model, run_elements): + """ Delegating the addition of results with run cache in the output + object to the appropriate handler.""" + self.handler.add_run_elements(model, run_elements) + + +class OutputHandlerInterface(metaclass=abc.ABCMeta): + """ + Interface for the creation of different output type handlers. + """ + @classmethod + def __subclasshook__(cls, subclass): + return (hasattr(subclass, 'initialize') and + callable(subclass.initialize) and + hasattr(subclass, 'update') and + callable(subclass.update) and + hasattr(subclass, 'postprocess') and + callable(subclass.postprocess) and + hasattr(subclass, 'add_run_elements') and + callable(subclass.add_run_elements) or + NotImplemented) + + @abc.abstractmethod + def initialize(self, model, capture_elements): + """ + Create the results object and its elements based on capture_elemetns. + """ + raise NotImplementedError + + def update(self, model, capture_elements): + """ + Update the results object at each iteration at which resutls are + stored. + """ + raise NotImplementedError + + def postprocess(self, **kwargs): + """ + Perform different tasks at the time of returning the results object. + """ + raise NotImplementedError + + def add_run_elements(self, model, capture_elements): + """ + Add elements with run cache to the results object. + """ + raise NotImplementedError + + +class DatasetHandler(OutputHandlerInterface): + """ + Manages simulation results stored as netCDF4 Dataset. + """ + + def __init__(self, out_file): + self.out_file = out_file + self.step = 0 + self.ds = None + + def initialize(self, model, capture_elements): + """ + Creates a netCDF4 Dataset and adds model dimensions and variables + present in the capture elements to it. + + Parameters + ---------- + model: pysd.Model + PySD Model object + capture_elements: set + Which model elements to capture - uses pysafe names. + + Returns + ------- + None + + """ + self.ds = nc.Dataset(self.out_file, "w") + # defining global attributes + self.ds.description = "Results for simulation run on" \ + f"{t.ctime(t.time())} using PySD version {__version__}" + self.ds.model_file = model.py_model_file + self.ds.timestep = f"{model.components.time_step()}" + self.ds.initial_time = f"{model.components.initial_time()}" + self.ds.final_time = f"{model.components.final_time()}" + + # creating variables for all model dimensions + for dim_name, coords in model.subscripts.items(): + coords = np.array(coords) + # create dimension + self.ds.createDimension(dim_name, len(coords)) + # length of the longest string in the coords + max_str_len = len(max(coords, key=len)) + + # create variable + # TODO: check if the type could be defined otherwise) + var = self.ds.createVariable(dim_name, + f"S{max_str_len}", + (dim_name,)) + # assigning values to variable + var[:] = coords + + # creating the time dimension as unlimited + self.ds.createDimension("time", None) + + # creating variables in capture_elements + self.__create_ds_vars(model, capture_elements) + + def update(self, model, capture_elements): + """ + Writes values of variables in capture_elements in the netCDF4 Dataset. + + Parameters + ---------- + model: pysd.Model + PySD Model object + capture_elements: set + Which model elements to capture - uses pysafe names. + + Returns + ------- + None + """ + for key in capture_elements: + comp = getattr(model.components, key) + comp_vals = comp() + if isinstance(comp_vals, xr.DataArray): + self.ds[key][self.step, :] = comp_vals.values + elif isinstance(comp_vals, np.ndarray): + self.ds[key][self.step, :] = comp_vals + else: + self.ds[key][self.step] = comp_vals + + self.step += 1 + + def postprocess(self, **kwargs): + """ + Closes netCDF4 Dataset. + + Returns + ------- + None + """ + self.ds.close() + + if kwargs.get("flatten"): + warnings.warn("DataArrays stored in netCDF4 will not be flattened") + + print(f"Results stored in {self.out_file}") + + def add_run_elements(self, model, capture_elements): + """ + Adds constant elements to netCDF4 Dataset. + + Parameters + ---------- + model: pysd.Model + PySD Model object + capture_elements: list + List of constant elements + + Returns + ------- + None + """ + # creating variables in capture_elements + # TODO we are looping through all capture elements twice. This + # could be avoided + self.__create_ds_vars(model, capture_elements) + + for key in capture_elements: + comp = getattr(model.components, key) + comp_vals = comp() + try: + for num, _ in enumerate(self.ds["time"][:]): + if isinstance(comp_vals, xr.DataArray): + self.ds[key][num, :] = comp_vals.values + elif isinstance(comp_vals, np.ndarray): + if comp_vals.size == 1: + self.ds[key][num] = comp_vals + else: + self.ds[key][num, :] = comp_vals + else: + self.ds[key][num] = comp_vals + except ValueError: + warnings.warn(f"The dimensions of {key} in the results " + "do not match the declared dimensions for this " + "variable. The resulting values will not be " + "included in the results file.") + + def __create_ds_vars(self, model, capture_elements): + """ + Create new variables in a netCDF4 Dataset from the capture_elements. + + Parameters + ---------- + model: pysd.Model + PySD Model object + capture_elements: set + Which model elements to capture - uses pysafe names. + + Returns + ------- + None + + """ + + for key in capture_elements: + comp = getattr(model.components, key) + comp_vals = comp() + + if isinstance(comp_vals, (xr.DataArray, np.ndarray)): + if comp.subscripts: + dims = ("time",) + tuple(comp.subscripts) + else: + dims = ("time",) + else: + dims = ("time",) + + self.ds.createVariable(key, "f8", dims) + + # adding units and description as metadata for each var + self.ds[key].units = model.doc.loc[ + model.doc["Py Name"] == key, + "Units"].values[0] or "Missing" + self.ds[key].description = model.doc.loc[ + model.doc["Py Name"] == key, + "Comment"].values[0] or "Missing" + + +class DataFrameHandler(OutputHandlerInterface): + """ + Manages simulation results stored as pandas DataFrame. + """ + def __init__(self): + self.ds = None + + def initialize(self, model, capture_elements): + """ + Creates a pandas DataFrame and adds model variables as columns. + + Parameters + ---------- + model: pysd.Model + PySD Model object + capture_elements: set + Which model elements to capture - uses pysafe names. + + Returns + ------- + None + + """ + self.ds = pd.DataFrame(columns=capture_elements) + + def update(self, model, capture_elements): + """ + Add a row to the results pandas DataFrame with the values of the + variables listed in capture_elements. + + Parameters + ---------- + model: pysd.Model + PySD Model object + capture_elements: set + Which model elements to capture - uses pysafe names. + + Returns + ------- + None + """ + + self.ds.at[model.time.round()] = [ + getattr(model.components, key)() + for key in capture_elements] + + def postprocess(self, **kwargs): + """ + Delete time column from the pandas DataFrame and flatten xarrays if + required. + + Returns + ------- + ds: pandas.DataFrame + Simulation results stored as a pandas DataFrame. + """ + # delete time column as it was created only for avoiding errors + # of appending data. See previous TODO. + del self.ds["time"] + + return utils.make_flat_df(self.ds, + kwargs["return_addresses"], + kwargs["flatten"]) + + def add_run_elements(self, model, capture_elements): + """ + Adds constant elements to a dataframe. + + Parameters + ---------- + model: pysd.Model + PySD Model object + capture_elements: set + Which model elements to capture - uses pysafe names. + + Returns + ------- + None + """ + nx = len(self.ds.index) + for element in capture_elements: + self.ds[element] = [getattr(model.components, element)()] * nx diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index 54f81cde..12ebb3c3 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -1195,11 +1195,11 @@ def test_get_series_data(self): def test__integrate(self, shared_tmpdir): from pysd.py_backend.model import ModelOutput - # Todo: think through a stronger test here... + # TODO: think through a stronger test here... model = pysd.read_vensim(test_model) model.progress = False model.time.add_return_timestamps(list(range(0, 5, 2))) - capture_elements={'teacup_temperature'} + capture_elements = {'teacup_temperature'} out = ModelOutput(model, capture_elements, None) model._integrate(out) From 786a3b0b81d01375a4fc3fafebf326cc3501f9ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Mon, 5 Sep 2022 12:08:24 +0200 Subject: [PATCH 12/43] remove time dimension from run cache elements --- pysd/py_backend/model.py | 20 +++++----- pysd/py_backend/output.py | 78 +++++++++++++++++++++------------------ 2 files changed, 53 insertions(+), 45 deletions(-) diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index 6ee741f1..2b2195b6 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -1161,6 +1161,7 @@ def run(self, params=None, return_columns=None, return_timestamps=None, # create a dictionary splitting run cached and others capture_elements = self._split_capture_elements(capture_elements) + # include outputs in cache if needed self._dependencies["OUTPUTS"] = { element: 1 for element in capture_elements["step"] @@ -1170,20 +1171,21 @@ def run(self, params=None, return_columns=None, return_timestamps=None, # udate the cache type taking into account the outputs self._assign_cache_type() - # check validitty of output_file + # check validitty of output_file. This could be done inside the + # ModelOutput class, but it feels too late if output_file: - if isinstance(output_file, str): - output_file = Path(output_file) - ext = output_file.suffix - elif isinstance(output_file, Path): - ext = output_file.suffix - else: + if not isinstance(output_file, (str, Path)): raise TypeError( "Paths must be strings or pathlib Path objects.") - if ext not in ModelOutput.valid_output_files: + if isinstance(output_file, str): + output_file = Path(output_file) + + file_extension = output_file.suffix + + if file_extension not in ModelOutput.valid_output_files: raise ValueError( - f"Unsupported output file format {ext}") + f"Unsupported output file format {file_extension}") # add constant cache to thosa variable that are constants self._add_constant_cache() diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index f5aa55dd..f589c282 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -145,6 +145,7 @@ def initialize(self, model, capture_elements): """ self.ds = nc.Dataset(self.out_file, "w") + # defining global attributes self.ds.description = "Results for simulation run on" \ f"{t.ctime(t.time())} using PySD version {__version__}" @@ -163,9 +164,8 @@ def initialize(self, model, capture_elements): # create variable # TODO: check if the type could be defined otherwise) - var = self.ds.createVariable(dim_name, - f"S{max_str_len}", - (dim_name,)) + var = self.ds.createVariable(dim_name, f"S{max_str_len}", + (dim_name,)) # assigning values to variable var[:] = coords @@ -191,14 +191,33 @@ def update(self, model, capture_elements): None """ for key in capture_elements: + comp = getattr(model.components, key) comp_vals = comp() - if isinstance(comp_vals, xr.DataArray): - self.ds[key][self.step, :] = comp_vals.values - elif isinstance(comp_vals, np.ndarray): - self.ds[key][self.step, :] = comp_vals + + if "time" in self.ds[key].dimensions: + if isinstance(comp_vals, xr.DataArray): + self.ds[key][self.step, :] = comp_vals.values + elif isinstance(comp_vals, np.ndarray): + self.ds[key][self.step, :] = comp_vals + else: + self.ds[key][self.step] = comp_vals else: - self.ds[key][self.step] = comp_vals + try: # this issue can arise with external objects + if isinstance(comp_vals, xr.DataArray): + self.ds[key][:] = comp_vals.values + elif isinstance(comp_vals, np.ndarray): + if comp_vals.size == 1: + self.ds[key][:] = comp_vals + else: + self.ds[key][:] = comp_vals + else: + self.ds[key][:] = comp_vals + except ValueError: + warnings.warn(f"The dimensions of {key} in the results " + "do not match the declared dimensions for this " + "variable. The resulting values will not be " + "included in the results file.") self.step += 1 @@ -210,6 +229,8 @@ def postprocess(self, **kwargs): ------- None """ + + # close Dataset self.ds.close() if kwargs.get("flatten"): @@ -235,29 +256,11 @@ def add_run_elements(self, model, capture_elements): # creating variables in capture_elements # TODO we are looping through all capture elements twice. This # could be avoided - self.__create_ds_vars(model, capture_elements) + self.__create_ds_vars(model, capture_elements, add_time=False) - for key in capture_elements: - comp = getattr(model.components, key) - comp_vals = comp() - try: - for num, _ in enumerate(self.ds["time"][:]): - if isinstance(comp_vals, xr.DataArray): - self.ds[key][num, :] = comp_vals.values - elif isinstance(comp_vals, np.ndarray): - if comp_vals.size == 1: - self.ds[key][num] = comp_vals - else: - self.ds[key][num, :] = comp_vals - else: - self.ds[key][num] = comp_vals - except ValueError: - warnings.warn(f"The dimensions of {key} in the results " - "do not match the declared dimensions for this " - "variable. The resulting values will not be " - "included in the results file.") - - def __create_ds_vars(self, model, capture_elements): + self.update(model, capture_elements) + + def __create_ds_vars(self, model, capture_elements, add_time=True): """ Create new variables in a netCDF4 Dataset from the capture_elements. @@ -267,6 +270,8 @@ def __create_ds_vars(self, model, capture_elements): PySD Model object capture_elements: set Which model elements to capture - uses pysafe names. + add_time: bool + Whether to add a time as the first dimension for the variables. Returns ------- @@ -278,15 +283,16 @@ def __create_ds_vars(self, model, capture_elements): comp = getattr(model.components, key) comp_vals = comp() + dims = () + if isinstance(comp_vals, (xr.DataArray, np.ndarray)): if comp.subscripts: - dims = ("time",) + tuple(comp.subscripts) - else: - dims = ("time",) - else: - dims = ("time",) + dims = tuple(comp.subscripts) + + if add_time: + dims = ("time",) + dims - self.ds.createVariable(key, "f8", dims) + self.ds.createVariable(key, "f8", dims, compression="zlib") # adding units and description as metadata for each var self.ds[key].units = model.doc.loc[ From 4ce0221892849ee0461965324a7bc0a74dc02b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Mon, 5 Sep 2022 14:09:08 +0200 Subject: [PATCH 13/43] allow .nc exports from cli --- docs/command_line_usage.rst | 6 ++-- pysd/cli/main.py | 51 ++++++------------------------ pysd/cli/parser.py | 5 +-- pysd/py_backend/model.py | 6 ++-- pysd/py_backend/output.py | 55 ++++++++++++++++++++++++++++----- tests/pytest_pysd/pytest_cli.py | 2 +- 6 files changed, 68 insertions(+), 57 deletions(-) diff --git a/docs/command_line_usage.rst b/docs/command_line_usage.rst index 385a44e7..b959ee8a 100644 --- a/docs/command_line_usage.rst +++ b/docs/command_line_usage.rst @@ -26,11 +26,11 @@ In order to set the output file path, the *-o/--output-file* argument can be use python -m pysd -o my_output_file.csv Teacup.mdl .. note:: - The output file can be a *.csv* or *.tab*. + The output file format may be *.csv*, *.tab* or *.nc*. .. note:: - If *-o/--output-file* is not given, the output will be saved in a file - that starts with the model file name followed by a time stamp to avoid + If *-o/--output-file* is not given, the output will be saved in a *.tab* + file that starts with the model file name followed by a time stamp to avoid overwritting files. Activate progress bar diff --git a/pysd/cli/main.py b/pysd/cli/main.py index 29601b52..fb28940e 100644 --- a/pysd/cli/main.py +++ b/pysd/cli/main.py @@ -2,17 +2,15 @@ import os from pathlib import Path -from csv import QUOTE_NONE from datetime import datetime -from .parser import parser - import pysd from pysd.translators.vensim.vensim_utils import supported_extensions as\ vensim_extensions from pysd.translators.xmile.xmile_utils import supported_extensions as\ xmile_extensions +from .parser import parser def main(args): """ @@ -41,12 +39,17 @@ def main(args): model.initialize() - output = model.run(**create_configuration(model, options)) + if not options.output_file: + options.output_file = os.path.splitext(os.path.basename( + options.model_file + ))[0]\ + + datetime.now().strftime("_output_%Y_%m_%d-%H_%M_%S_%f.tab") + + model.run(**create_configuration(model, options)) if options.export_file: model.export(options.export_file) - save(output, options) print("\nFinished!") sys.exit() @@ -133,45 +136,11 @@ def create_configuration(model, options): "time_step": options.time_step, "saveper": options.saveper, "flatten_output": True, # need to return totally flat DF - "return_timestamps": options.return_timestamps # given or None + "return_timestamps": options.return_timestamps, # given or None, + "output_file": options.output_file } if options.import_file: conf_dict["initial_condition"] = options.import_file return conf_dict - - -def save(output, options): - """ - Saves models output. - - Paramters - --------- - output: pandas.DataFrame - - options: argparse.Namespace - - Returns - ------- - None - - """ - if options.output_file: - output_file = options.output_file - else: - output_file = os.path.splitext(os.path.basename( - options.model_file - ))[0]\ - + datetime.now().strftime("_output_%Y_%m_%d-%H_%M_%S_%f.tab") - - if output_file.endswith(".tab"): - sep = "\t" - else: - sep = "," - - # QUOTE_NONE used to print the csv/tab files af vensim does with special - # characterse, e.g.: "my-var"[Dimension] - output.to_csv(output_file, sep, index_label="Time", quoting=QUOTE_NONE) - - print(f"Data saved in '{output_file}'") diff --git a/pysd/cli/parser.py b/pysd/cli/parser.py index dbe6e012..45dec12c 100644 --- a/pysd/cli/parser.py +++ b/pysd/cli/parser.py @@ -29,10 +29,11 @@ def check_output(string): Checks that out put file ends with .tab or .csv """ - if not string.endswith('.tab') and not string.endswith('.csv'): + if not string.endswith('.tab') and not string.endswith('.csv') and not \ + string.endswith('.nc'): parser.error( f'when parsing {string}' - '\nThe output file name must be .tab or .csv...') + '\nThe output file name must be .tab, .csv or .nc...') return string diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index 2b2195b6..f75ae4fd 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -1120,6 +1120,8 @@ def run(self, params=None, return_columns=None, return_timestamps=None, >>> model.run(return_timestamps=[1, 2, 3, 4, 10]) >>> model.run(return_timestamps=10) >>> model.run(return_timestamps=np.linspace(1, 10, 20)) + >>> model.run(output_file="results.nc") + See Also -------- @@ -1206,8 +1208,8 @@ def run(self, params=None, return_columns=None, return_timestamps=None, self._remove_constant_cache() - return output.postprocess(return_addresses=return_addresses, - flatten=flatten_output) + return output.postprocess( + return_addresses=return_addresses, flatten=flatten_output) def select_submodel(self, vars=[], modules=[], exogenous_components={}): """ diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index f589c282..ddafcb6f 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -9,6 +9,8 @@ import warnings import time as t +from csv import QUOTE_NONE + import numpy as np import xarray as xr import pandas as pd @@ -33,7 +35,7 @@ class ModelOutput(): out_file: str or pathlib.Path Path to the file where the results will be written. """ - valid_output_files = [".nc"] + valid_output_files = [".nc", ".csv", ".tab"] def __init__(self, model, capture_elements, out_file=None): @@ -49,9 +51,9 @@ def __handle(self, out_file): if out_file: if out_file.suffix == ".nc": return DatasetHandler(out_file) - raise ValueError( - f"Unsupported output file format {out_file.suffix}") - return DataFrameHandler() + # when the users expects a csv or tab output file, it defaults to the + # DataFrame path + return DataFrameHandler(out_file) def initialize(self, model): """ Delegating the creation of the results object and its elements to @@ -307,8 +309,9 @@ class DataFrameHandler(OutputHandlerInterface): """ Manages simulation results stored as pandas DataFrame. """ - def __init__(self): + def __init__(self, out_file): self.ds = None + self.output_file = out_file def initialize(self, model, capture_elements): """ @@ -363,9 +366,45 @@ def postprocess(self, **kwargs): # of appending data. See previous TODO. del self.ds["time"] - return utils.make_flat_df(self.ds, - kwargs["return_addresses"], - kwargs["flatten"]) + # enforce flattening if df is to be saved to csv or tab file + flatten = True if self.output_file else kwargs.get("flatten", None) + + df = utils.make_flat_df(self.ds, + kwargs["return_addresses"], + flatten) + if self.output_file: + self.__save_to_file(df) + + return df + + def __save_to_file(self, output): + """ + Saves models output. + + Paramters + --------- + output: pandas.DataFrame + + options: argparse.Namespace + + Returns + ------- + None + + """ + + if self.output_file.suffix == ".tab": + sep = "\t" + else: + sep = "," + + # QUOTE_NONE used to print the csv/tab files as vensim does with special + # characterse, e.g.: "my-var"[Dimension] + output.to_csv( + self.output_file, sep, index_label="Time", quoting=QUOTE_NONE + ) + + print(f"Data saved in '{self.output_file}'") def add_run_elements(self, model, capture_elements): """ diff --git a/tests/pytest_pysd/pytest_cli.py b/tests/pytest_pysd/pytest_cli.py index 636074ab..7770d247 100644 --- a/tests/pytest_pysd/pytest_cli.py +++ b/tests/pytest_pysd/pytest_cli.py @@ -100,7 +100,7 @@ def test_read_not_valid_output(self, _root): stderr = out.stderr.decode(encoding_stderr) assert out.returncode != 0 assert f"PySD: error: when parsing {out_xls_file}" in stderr - assert "The output file name must be .tab or .csv..." in stderr + assert "The output file name must be .tab, .csv or .nc..." in stderr def test_read_not_valid_time_stamps(self): From 478ff22d5a4b652495888167f994637628e1a22a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Mon, 5 Sep 2022 17:22:13 +0200 Subject: [PATCH 14/43] increase coverage --- pysd/py_backend/output.py | 3 + tests/pytest_pysd/pytest_pysd.py | 117 +++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index ddafcb6f..a3b96c48 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -99,6 +99,7 @@ def initialize(self, model, capture_elements): """ raise NotImplementedError + @abc.abstractmethod def update(self, model, capture_elements): """ Update the results object at each iteration at which resutls are @@ -106,12 +107,14 @@ def update(self, model, capture_elements): """ raise NotImplementedError + @abc.abstractmethod def postprocess(self, **kwargs): """ Perform different tasks at the time of returning the results object. """ raise NotImplementedError + @abc.abstractmethod def add_run_elements(self, model, capture_elements): """ Add elements with run cache to the results object. diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index 12ebb3c3..ef88a98c 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -1680,3 +1680,120 @@ def test_run_export_import_initial(self): assert_frames_close(stocks2, stocks) + +class TestOutputs(): + + def test_output_handler_interface(self): + from pysd.py_backend.output import OutputHandlerInterface, \ + DatasetHandler, DataFrameHandler, ModelOutput + + # when the class does not inherit from OutputHandlerInterface, it must + # implement all the interface to be a subclass of + # OutputHandlerInterface. + # Add any additional Handler here. + assert issubclass(DatasetHandler, OutputHandlerInterface) + assert issubclass(DataFrameHandler, OutputHandlerInterface) + assert issubclass(ModelOutput, OutputHandlerInterface) + + class ThatFollowsInterface: + """ + This class does not inherit from OutputHandlerInterface, but it + overrides all its methods (it follows the interface). + """ + + def initialize(self, model, capture_elements): + pass + + def update(self, model, capture_elements): + pass + + def postprocess(self, **kwargs): + pass + + def add_run_elements(self, capture_elemetns): + pass + + # eventhough it does not inherit from OutputHandlerInterface, it is + # considered a subclass, because it follows the interface + assert issubclass(ThatFollowsInterface, OutputHandlerInterface) + + + class IncompleteHandler: + """ + Class that does not follow the full interface + (add_run_elements is missing). + """ + def initialize(self, model, capture_elements): + pass + + def update(self, model, capture_elements): + pass + + def postprocess(self, **kwargs): + pass + + # It does not inherit from OutputHandlerInterface and does not fulfill + # its interface + assert issubclass(IncompleteHandler, OutputHandlerInterface) == False + + + class EmptyHandler(OutputHandlerInterface): + """ + When the class DOES inherit from OutputHandlerInterface, but does + not override all its abstract methods, then it cannot be + instantiated + """ + pass + + # it is a subclass because it inherits from it + assert issubclass(EmptyHandler, OutputHandlerInterface) + + # it cannot be instantiated because it does not override all abstract + # methods + with pytest.raises(TypeError): + empty = EmptyHandler() + + # calling methods that are not overriden returns NotImplementedError + # this should never happen, because these methods are instance methods, + # therefore the class needs to be instantiated first + with pytest.raises(NotImplementedError): + EmptyHandler.initialize(EmptyHandler, "model", "capture") + + with pytest.raises(NotImplementedError): + EmptyHandler.update(EmptyHandler, "model", "capture") + + with pytest.raises(NotImplementedError): + EmptyHandler.postprocess(EmptyHandler) + + with pytest.raises(NotImplementedError): + EmptyHandler.add_run_elements( + EmptyHandler, "model", "capture") + + + def test_output_with_dimensions(self, shared_tmpdir): + model = pysd.read_vensim(test_model_look) + model.progress = False + + out_file = shared_tmpdir.joinpath("results.nc") + + with catch_warnings(record=True) as w: + simplefilter("always") + model.run(output_file=out_file) + + with nc.Dataset(out_file, "r")as ds: + assert ds.ncattrs() == ['description', 'model_file', 'timestep', + 'initial_time', 'final_time'] + assert list(ds.dimensions.keys()) == ["Rows", "Dim", "time"] + # dimensions are stored as variables + assert ds["Rows"][:].size == 2 + assert "Rows" in ds.variables.keys() + assert "time" in ds.variables.keys() + # scalars do not have the time dimension + assert ds["initial_time"][:].size == 1 + # cache step variables have the "time" dimension + assert ds["lookup_1d_time"].dimensions == ("time",) + + assert ds["d2d"].dimensions == ("time", "Rows", "Dim") + assert ds["d2d"].description == "Missing" + assert ds["d2d"].units == "Missing" + From 94789ff2831ed257fd801995df67a5b5c4c803c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Mon, 5 Sep 2022 19:54:14 +0200 Subject: [PATCH 15/43] move make_flat_df and tests --- pysd/py_backend/output.py | 118 ++++++++++++++-- pysd/py_backend/utils.py | 87 ------------ tests/pytest_pysd/pytest_pysd.py | 225 ++++++++++++++++++++++++++++-- tests/pytest_pysd/pytest_utils.py | 191 ------------------------- 4 files changed, 318 insertions(+), 303 deletions(-) diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index a3b96c48..42b7db65 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -11,6 +11,8 @@ from csv import QUOTE_NONE +import regex as re + import numpy as np import xarray as xr import pandas as pd @@ -18,7 +20,7 @@ from pysd._version import __version__ -from . import utils +from . utils import xrsplit class ModelOutput(): @@ -169,8 +171,8 @@ def initialize(self, model, capture_elements): # create variable # TODO: check if the type could be defined otherwise) - var = self.ds.createVariable(dim_name, f"S{max_str_len}", - (dim_name,)) + var = self.ds.createVariable( + dim_name, f"S{max_str_len}", (dim_name,)) # assigning values to variable var[:] = coords @@ -208,7 +210,7 @@ def update(self, model, capture_elements): else: self.ds[key][self.step] = comp_vals else: - try: # this issue can arise with external objects + try: # this issue can arise with external objects if isinstance(comp_vals, xr.DataArray): self.ds[key][:] = comp_vals.values elif isinstance(comp_vals, np.ndarray): @@ -219,10 +221,11 @@ def update(self, model, capture_elements): else: self.ds[key][:] = comp_vals except ValueError: - warnings.warn(f"The dimensions of {key} in the results " - "do not match the declared dimensions for this " - "variable. The resulting values will not be " - "included in the results file.") + warnings.warn( + f"The dimensions of {key} in the results " + "do not match the declared dimensions for this " + "variable. The resulting values will not be " + "included in the results file.") self.step += 1 @@ -332,7 +335,7 @@ def initialize(self, model, capture_elements): None """ - self.ds = pd.DataFrame(columns=capture_elements) + self.ds = pd.DataFrame(columns=capture_elements) def update(self, model, capture_elements): """ @@ -372,9 +375,9 @@ def postprocess(self, **kwargs): # enforce flattening if df is to be saved to csv or tab file flatten = True if self.output_file else kwargs.get("flatten", None) - df = utils.make_flat_df(self.ds, - kwargs["return_addresses"], - flatten) + df = DataFrameHandler.make_flat_df( + self.ds, kwargs["return_addresses"], flatten + ) if self.output_file: self.__save_to_file(df) @@ -401,8 +404,8 @@ def __save_to_file(self, output): else: sep = "," - # QUOTE_NONE used to print the csv/tab files as vensim does with special - # characterse, e.g.: "my-var"[Dimension] + # QUOTE_NONE used to print the csv/tab files as vensim does with + # special characterse, e.g.: "my-var"[Dimension] output.to_csv( self.output_file, sep, index_label="Time", quoting=QUOTE_NONE ) @@ -427,3 +430,90 @@ def add_run_elements(self, model, capture_elements): nx = len(self.ds.index) for element in capture_elements: self.ds[element] = [getattr(model.components, element)()] * nx + + @staticmethod + def make_flat_df(df, return_addresses, flatten=False): + """ + Takes a dataframe from the outputs of the integration processes, + renames the columns as the given return_adresses and splits xarrays + if needed. + + Parameters + ---------- + df: pandas.DataFrame + Dataframe to process. + + return_addresses: dict + Keys will be column names of the resulting dataframe, and are what the + user passed in as 'return_columns'. Values are a tuple: + (py_name, {coords dictionary}) which tells us where to look for the + value to put in that specific column. + + flatten: bool (optional) + If True, once the output dataframe has been formatted will + split the xarrays in new columns following vensim's naming + to make a totally flat output. Default is False. + + Returns + ------- + new_df: pandas.DataFrame + Formatted dataframe. + + """ + new_df = {} + for real_name, (pyname, address) in return_addresses.items(): + if address: + # subset the specific address + values = [x.loc[address] for x in df[pyname].values] + else: + # get the full column + values = df[pyname].to_list() + + is_dataarray = len(values) != 0 and isinstance( + values[0], xr.DataArray) + + if is_dataarray and values[0].size == 1: + # some elements are returned as 0-d arrays, convert + # them to float + values = [float(x) for x in values] + is_dataarray = False + + if flatten and is_dataarray: + DataFrameHandler.__add_flat(new_df, real_name, values) + else: + new_df[real_name] = values + + return pd.DataFrame(index=df.index, data=new_df) + + @staticmethod + def __add_flat(savedict, name, values): + """ + Add float lists from a list of xarrays to a provided dictionary. + + Parameters + ---------- + savedict: dict + Dictionary to save the data on. + + name: str + The base name of the variable to save the data. + + values: list + List of xarrays to convert to split in floats. + + Returns + ------- + None + + """ + # remove subscripts from name if given + name = re.sub(r'\[.*\]', '', name) + dims = values[0].dims + + # split values in xarray.DataArray + lval = [xrsplit(val) for val in values] + for i, ar in enumerate(lval[0]): + vals = [float(v[i]) for v in lval] + subs = '[' + ','.join([str(ar.coords[dim].values) + for dim in dims]) + ']' + savedict[name+subs] = vals diff --git a/pysd/py_backend/utils.py b/pysd/py_backend/utils.py index b31ebdae..6b8eb97c 100644 --- a/pysd/py_backend/utils.py +++ b/pysd/py_backend/utils.py @@ -9,7 +9,6 @@ from pathlib import Path from chardet.universaldetector import UniversalDetector -import regex as re import progressbar import numpy as np import xarray as xr @@ -104,92 +103,6 @@ def get_return_elements(return_columns, namespace): return list(capture_elements), return_addresses -def make_flat_df(df, return_addresses, flatten=False): - """ - Takes a dataframe from the outputs of the integration processes, - renames the columns as the given return_adresses and splits xarrays - if needed. - - Parameters - ---------- - df: Pandas.DataFrame - Output from the integration. - - return_addresses: dict - Keys will be column names of the resulting dataframe, and are what the - user passed in as 'return_columns'. Values are a tuple: - (py_name, {coords dictionary}) which tells us where to look for the - value to put in that specific column. - - flatten: bool (optional) - If True, once the output dataframe has been formatted will - split the xarrays in new columns following vensim's naming - to make a totally flat output. Default is False. - - Returns - ------- - new_df: pandas.DataFrame - Formatted dataframe. - - """ - new_df = {} - for real_name, (pyname, address) in return_addresses.items(): - if address: - # subset the specific address - values = [x.loc[address] for x in df[pyname].values] - else: - # get the full column - values = df[pyname].to_list() - - is_dataarray = len(values) != 0 and isinstance(values[0], xr.DataArray) - - if is_dataarray and values[0].size == 1: - # some elements are returned as 0-d arrays, convert - # them to float - values = [float(x) for x in values] - is_dataarray = False - - if flatten and is_dataarray: - _add_flat(new_df, real_name, values) - else: - new_df[real_name] = values - - return pd.DataFrame(index=df.index, data=new_df) - - -def _add_flat(savedict, name, values): - """ - Add float lists from a list of xarrays to a provided dictionary. - - Parameters - ---------- - savedict: dict - Dictionary to save the data on. - - name: str - The base name of the variable to save the data. - - values: list - List of xarrays to convert to split in floats. - - Returns - ------- - None - - """ - # remove subscripts from name if given - name = re.sub(r'\[.*\]', '', name) - dims = values[0].dims - - # split values in xarray.DataArray - lval = [xrsplit(val) for val in values] - for i, ar in enumerate(lval[0]): - vals = [float(v[i]) for v in lval] - subs = '[' + ','.join([str(ar.coords[dim].values) - for dim in dims]) + ']' - savedict[name+subs] = vals - - def compute_shape(coords, reshape_len=None, py_name=""): """ Computes the 'shape' of a coords dictionary. diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index ef88a98c..1c8cf91a 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -8,6 +8,8 @@ import netCDF4 as nc from pysd.tools.benchmarking import assert_frames_close +from pysd.py_backend.output import OutputHandlerInterface, DatasetHandler, \ + DataFrameHandler, ModelOutput import pysd @@ -24,13 +26,15 @@ + "test_get_lookups_subscripted_args.mdl") test_model_data = _root.joinpath( "test-models/tests/get_data_args_3d_xls/test_get_data_args_3d_xls.mdl") - +test_model_constants = _root.joinpath( + "test-models/tests/get_constants_subranges/" + "test_get_constants_subranges.mdl" +) more_tests = _root.joinpath("more-tests/") test_model_constant_pipe = more_tests.joinpath( "constant_pipeline/test_constant_pipeline.mdl") - class TestPySD(): def test_run(self): @@ -1684,9 +1688,6 @@ def test_run_export_import_initial(self): class TestOutputs(): def test_output_handler_interface(self): - from pysd.py_backend.output import OutputHandlerInterface, \ - DatasetHandler, DataFrameHandler, ModelOutput - # when the class does not inherit from OutputHandlerInterface, it must # implement all the interface to be a subclass of # OutputHandlerInterface. @@ -1717,7 +1718,6 @@ def add_run_elements(self, capture_elemetns): # considered a subclass, because it follows the interface assert issubclass(ThatFollowsInterface, OutputHandlerInterface) - class IncompleteHandler: """ Class that does not follow the full interface @@ -1734,8 +1734,7 @@ def postprocess(self, **kwargs): # It does not inherit from OutputHandlerInterface and does not fulfill # its interface - assert issubclass(IncompleteHandler, OutputHandlerInterface) == False - + assert not issubclass(IncompleteHandler, OutputHandlerInterface) class EmptyHandler(OutputHandlerInterface): """ @@ -1769,7 +1768,6 @@ class EmptyHandler(OutputHandlerInterface): EmptyHandler.add_run_elements( EmptyHandler, "model", "capture") - def test_output_with_dimensions(self, shared_tmpdir): model = pysd.read_vensim(test_model_look) model.progress = False @@ -1781,8 +1779,9 @@ def test_output_with_dimensions(self, shared_tmpdir): model.run(output_file=out_file) with nc.Dataset(out_file, "r")as ds: - assert ds.ncattrs() == ['description', 'model_file', 'timestep', - 'initial_time', 'final_time'] + assert ds.ncattrs() == [ + 'description', 'model_file', 'timestep', 'initial_time', + 'final_time'] assert list(ds.dimensions.keys()) == ["Rows", "Dim", "time"] # dimensions are stored as variables assert ds["Rows"][:].size == 2 @@ -1797,3 +1796,207 @@ def test_output_with_dimensions(self, shared_tmpdir): assert ds["d2d"].description == "Missing" assert ds["d2d"].units == "Missing" + # test cache run variables with dimensions + model2 = pysd.read_vensim(test_model_constants) + model2.progress = False + + out_file2 = shared_tmpdir.joinpath("results2.nc") + + with catch_warnings(record=True) as w: + simplefilter("always") + model2.run(output_file=out_file2) + + with nc.Dataset(out_file2, "r")as ds: + assert "constant" in list(ds.variables.keys()) + assert ds["constant"].dimensions == ("dim1",) + + def test_make_flat_df(self): + + df = pd.DataFrame(index=[1], columns=['elem1']) + df.at[1] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2'])] + + expected = pd.DataFrame(index=[1], data={'Elem1[B,F]': 6.}) + + return_addresses = { + 'Elem1[B,F]': ('elem1', {'Dim1': ['B'], 'Dim2': ['F']})} + + actual = DataFrameHandler.make_flat_df(df, return_addresses) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + assert_frames_close(actual, expected, rtol=1e-8, atol=1e-8) + + def test_make_flat_df_0dxarray(self): + + df = pd.DataFrame(index=[1], columns=['elem1']) + df.at[1] = [xr.DataArray(5)] + + expected = pd.DataFrame(index=[1], data={'Elem1': 5.}) + + return_addresses = {'Elem1': ('elem1', {})} + + actual = DataFrameHandler.make_flat_df(df, return_addresses, flatten=True) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + assert_frames_close(actual, expected, rtol=1e-8, atol=1e-8) + + def test_make_flat_df_nosubs(self): + + df = pd.DataFrame(index=[1], columns=['elem1', 'elem2']) + df.at[1] = [25, 13] + + expected = pd.DataFrame(index=[1], columns=['Elem1', 'Elem2']) + expected.at[1] = [25, 13] + + return_addresses = {'Elem1': ('elem1', {}), + 'Elem2': ('elem2', {})} + + actual = DataFrameHandler.make_flat_df(df, return_addresses) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + assert all(actual['Elem1'] == expected['Elem1']) + assert all(actual['Elem2'] == expected['Elem2']) + + def test_make_flat_df_return_array(self): + """ There could be cases where we want to + return a whole section of an array - ie, by passing in only part of + the simulation dictionary. in this case, we can't force to float...""" + + df = pd.DataFrame(index=[1], columns=['elem1', 'elem2']) + df.at[1] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2']), + xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2'])] + + expected = pd.DataFrame(index=[1], columns=['Elem1[A, Dim2]', 'Elem2']) + expected.at[1] = [xr.DataArray([[1, 2, 3]], + {'Dim1': ['A'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2']), + xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2'])] + + return_addresses = { + 'Elem1[A, Dim2]': ('elem1', {'Dim1': ['A'], + 'Dim2': ['D', 'E', 'F']}), + 'Elem2': ('elem2', {})} + + actual = DataFrameHandler.make_flat_df(df, return_addresses) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + # need to assert one by one as they are xarrays + assert actual.loc[1, 'Elem1[A, Dim2]'].equals( + expected.loc[1, 'Elem1[A, Dim2]']) + assert actual.loc[1, 'Elem2'].equals(expected.loc[1, 'Elem2']) + + def test_make_flat_df_flatten(self): + + df = pd.DataFrame(index=[1], columns=['elem1', 'elem2']) + df.at[1] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2']), + xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2'])] + + expected = pd.DataFrame(index=[1], columns=[ + 'Elem1[A,D]', + 'Elem1[A,E]', + 'Elem1[A,F]', + 'Elem2[A,D]', + 'Elem2[A,E]', + 'Elem2[A,F]', + 'Elem2[B,D]', + 'Elem2[B,E]', + 'Elem2[B,F]', + 'Elem2[C,D]', + 'Elem2[C,E]', + 'Elem2[C,F]']) + + expected.at[1] = [1, 2, 3, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + return_addresses = { + 'Elem1[A,Dim2]': ('elem1', {'Dim1': ['A'], + 'Dim2': ['D', 'E', 'F']}), + 'Elem2': ('elem2', {})} + + actual = DataFrameHandler.make_flat_df(df, return_addresses, flatten=True) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + # need to assert one by one as they are xarrays + for col in set(expected.columns): + assert actual.loc[:, col].values == expected.loc[:, col].values + + def test_make_flat_df_flatten_transposed(self): + + df = pd.DataFrame(index=[1], columns=['elem2']) + df.at[1] = [ + xr.DataArray( + [[1, 4, 7], [2, 5, 8], [3, 6, 9]], + {'Dim2': ['D', 'E', 'F'], 'Dim1': ['A', 'B', 'C']}, + ['Dim2', 'Dim1'] + ).transpose("Dim1", "Dim2") + ] + + expected = pd.DataFrame(index=[1], columns=[ + 'Elem2[A,D]', + 'Elem2[A,E]', + 'Elem2[A,F]', + 'Elem2[B,D]', + 'Elem2[B,E]', + 'Elem2[B,F]', + 'Elem2[C,D]', + 'Elem2[C,E]', + 'Elem2[C,F]']) + + expected.at[1] = [1, 2, 3, 4, 5, 6, 7, 8, 9] + + return_addresses = { + 'Elem2': ('elem2', {})} + + actual = DataFrameHandler.make_flat_df(df, return_addresses, flatten=True) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + # need to assert one by one as they are xarrays + for col in set(expected.columns): + assert actual.loc[:, col].values == expected.loc[:, col].values + + def test_make_flat_df_times(self): + + df = pd.DataFrame(index=[1, 2], columns=['elem1']) + df['elem1'] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2']), + xr.DataArray([[2, 4, 6], [8, 10, 12], [14, 16, 19]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2'])] + + expected = pd.DataFrame([{'Elem1[B,F]': 6}, {'Elem1[B,F]': 12}]) + expected.index = [1, 2] + + return_addresses = {'Elem1[B,F]': ('elem1', {'Dim1': ['B'], + 'Dim2': ['F']})} + actual = DataFrameHandler.make_flat_df(df, return_addresses) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + assert set(actual.index) == set(expected.index) + assert all(actual['Elem1[B,F]'] == expected['Elem1[B,F]']) diff --git a/tests/pytest_pysd/pytest_utils.py b/tests/pytest_pysd/pytest_utils.py index 91d6dafb..a3210c07 100644 --- a/tests/pytest_pysd/pytest_utils.py +++ b/tests/pytest_pysd/pytest_utils.py @@ -99,197 +99,6 @@ def test_get_return_elements_not_found_error(self): ["inflow_a", "inflow_b", "inflow_c"], {'Inflow A': 'inflow_a', 'Inflow B': 'inflow_b'}) - def test_make_flat_df(self): - - df = pd.DataFrame(index=[1], columns=['elem1']) - df.at[1] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2'])] - - expected = pd.DataFrame(index=[1], data={'Elem1[B,F]': 6.}) - - return_addresses = { - 'Elem1[B,F]': ('elem1', {'Dim1': ['B'], 'Dim2': ['F']})} - - actual = pysd.utils.make_flat_df(df, return_addresses) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - assert_frames_close(actual, expected, rtol=1e-8, atol=1e-8) - - def test_make_flat_df_0dxarray(self): - - df = pd.DataFrame(index=[1], columns=['elem1']) - df.at[1] = [xr.DataArray(5)] - - expected = pd.DataFrame(index=[1], data={'Elem1': 5.}) - - return_addresses = {'Elem1': ('elem1', {})} - - actual = pysd.utils.make_flat_df(df, return_addresses, flatten=True) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - assert_frames_close(actual, expected, rtol=1e-8, atol=1e-8) - - def test_make_flat_df_nosubs(self): - - df = pd.DataFrame(index=[1], columns=['elem1', 'elem2']) - df.at[1] = [25, 13] - - expected = pd.DataFrame(index=[1], columns=['Elem1', 'Elem2']) - expected.at[1] = [25, 13] - - return_addresses = {'Elem1': ('elem1', {}), - 'Elem2': ('elem2', {})} - - actual = pysd.utils.make_flat_df(df, return_addresses) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - assert all(actual['Elem1'] == expected['Elem1']) - assert all(actual['Elem2'] == expected['Elem2']) - - def test_make_flat_df_return_array(self): - """ There could be cases where we want to - return a whole section of an array - ie, by passing in only part of - the simulation dictionary. in this case, we can't force to float...""" - - df = pd.DataFrame(index=[1], columns=['elem1', 'elem2']) - df.at[1] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2']), - xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2'])] - - expected = pd.DataFrame(index=[1], columns=['Elem1[A, Dim2]', 'Elem2']) - expected.at[1] = [xr.DataArray([[1, 2, 3]], - {'Dim1': ['A'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2']), - xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2'])] - - return_addresses = { - 'Elem1[A, Dim2]': ('elem1', {'Dim1': ['A'], - 'Dim2': ['D', 'E', 'F']}), - 'Elem2': ('elem2', {})} - - actual = pysd.utils.make_flat_df(df, return_addresses) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - # need to assert one by one as they are xarrays - assert actual.loc[1, 'Elem1[A, Dim2]'].equals( - expected.loc[1, 'Elem1[A, Dim2]']) - assert actual.loc[1, 'Elem2'].equals(expected.loc[1, 'Elem2']) - - def test_make_flat_df_flatten(self): - - df = pd.DataFrame(index=[1], columns=['elem1', 'elem2']) - df.at[1] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2']), - xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2'])] - - expected = pd.DataFrame(index=[1], columns=[ - 'Elem1[A,D]', - 'Elem1[A,E]', - 'Elem1[A,F]', - 'Elem2[A,D]', - 'Elem2[A,E]', - 'Elem2[A,F]', - 'Elem2[B,D]', - 'Elem2[B,E]', - 'Elem2[B,F]', - 'Elem2[C,D]', - 'Elem2[C,E]', - 'Elem2[C,F]']) - - expected.at[1] = [1, 2, 3, 1, 2, 3, 4, 5, 6, 7, 8, 9] - - return_addresses = { - 'Elem1[A,Dim2]': ('elem1', {'Dim1': ['A'], - 'Dim2': ['D', 'E', 'F']}), - 'Elem2': ('elem2', {})} - - actual = pysd.utils.make_flat_df(df, return_addresses, flatten=True) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - # need to assert one by one as they are xarrays - for col in set(expected.columns): - assert actual.loc[:, col].values == expected.loc[:, col].values - - def test_make_flat_df_flatten_transposed(self): - - df = pd.DataFrame(index=[1], columns=['elem2']) - df.at[1] = [ - xr.DataArray( - [[1, 4, 7], [2, 5, 8], [3, 6, 9]], - {'Dim2': ['D', 'E', 'F'], 'Dim1': ['A', 'B', 'C']}, - ['Dim2', 'Dim1'] - ).transpose("Dim1", "Dim2") - ] - - expected = pd.DataFrame(index=[1], columns=[ - 'Elem2[A,D]', - 'Elem2[A,E]', - 'Elem2[A,F]', - 'Elem2[B,D]', - 'Elem2[B,E]', - 'Elem2[B,F]', - 'Elem2[C,D]', - 'Elem2[C,E]', - 'Elem2[C,F]']) - - expected.at[1] = [1, 2, 3, 4, 5, 6, 7, 8, 9] - - return_addresses = { - 'Elem2': ('elem2', {})} - - actual = pysd.utils.make_flat_df(df, return_addresses, flatten=True) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - # need to assert one by one as they are xarrays - for col in set(expected.columns): - assert actual.loc[:, col].values == expected.loc[:, col].values - - def test_make_flat_df_times(self): - - df = pd.DataFrame(index=[1, 2], columns=['elem1']) - df['elem1'] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2']), - xr.DataArray([[2, 4, 6], [8, 10, 12], [14, 16, 19]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2'])] - - expected = pd.DataFrame([{'Elem1[B,F]': 6}, {'Elem1[B,F]': 12}]) - expected.index = [1, 2] - - return_addresses = {'Elem1[B,F]': ('elem1', {'Dim1': ['B'], - 'Dim2': ['F']})} - actual = pysd.utils.make_flat_df(df, return_addresses) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - assert set(actual.index) == set(expected.index) - assert all(actual['Elem1[B,F]'] == expected['Elem1[B,F]']) - def test_doctests(self): doctest.DocTestSuite(pysd.utils) From 8839b47b9fcee21c29659be3aa7fd53270ef9478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Tue, 6 Sep 2022 09:14:31 +0200 Subject: [PATCH 16/43] add section on output_file argument in the docs --- docs/getting_started.rst | 14 +++++++++++++- docs/installation.rst | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 865ecbd1..95add588 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -156,6 +156,18 @@ The subscripted variables, in general, will be returned as :py:class:`xarray.Dat >>> model.run(flatten_output=True) + +Storing simulation results on a file +------------------------------------ +Simulation results can be stored as *.csv*, *.tab* or *.nc* (netCDF4) files by defining the desired output file path in the `output_file` argument, when calling the :py:meth:`.run` method:: + + >>> model.run(output_file="results.nc") + +If the `output_file` is not set, the :py:meth:`.run` method will return a :py:class:`pandas.DataFrame`. + +For most cases, the *.tab* file format is the safest choice. It is preferable over the *.csv* format when the model includes subscripted variables. The *.nc* format is recommended for large models, and when the user wants to keep metadata such as variable units and description. + + Setting parameter values ------------------------ In some situations we may want to modify the parameters of the model to investigate its behavior under different assumptions. There are several ways to do this in PySD, but the :py:meth:`.run` method gives us a convenient method in the `params` keyword argument. @@ -174,7 +186,7 @@ If the parameter value to change is a subscripted variable (vector, matrix...), >>> model.run(params={'Subscripted var': 0}) -A partial :py:class:`xarray.DataArray` can be used. For example a new variable with ‘dim2’ but not ‘dim2’. In that case, the result will be repeated in the remaining dimensions:: +A partial :py:class:`xarray.DataArray` can be used. For example a new variable with ‘dim2’ but not ‘dim1’. In that case, the result will be repeated in the remaining dimensions:: >>> import xarray as xr >>> new_value = xr.DataArray([1, 5], {'dim2': [1, 2]}, ['dim2']) diff --git a/docs/installation.rst b/docs/installation.rst index f32a5027..6e86595a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -46,6 +46,7 @@ PySD builds on the core Python data analytics stack, and the following third par * Pandas * Parsimonious * xarray +* netCDF4 * xlrd * lxml * regex From 4f37a198397c281a6b1cec533c34a8f8bc714dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Tue, 6 Sep 2022 09:57:41 +0200 Subject: [PATCH 17/43] add more tests --- pysd/cli/parser.py | 2 +- pysd/py_backend/model.py | 2 + pysd/py_backend/output.py | 13 +-- .../test_subscript_1d_arrays.mdl | 93 +++++++++++++++++++ tests/pytest_pysd/pytest_pysd.py | 33 ++++++- 5 files changed, 130 insertions(+), 13 deletions(-) create mode 100644 tests/more-tests/numeric_subscripts/test_subscript_1d_arrays.mdl diff --git a/pysd/cli/parser.py b/pysd/cli/parser.py index 45dec12c..707d91f4 100644 --- a/pysd/cli/parser.py +++ b/pysd/cli/parser.py @@ -30,7 +30,7 @@ def check_output(string): """ if not string.endswith('.tab') and not string.endswith('.csv') and not \ - string.endswith('.nc'): + string.endswith('.nc'): parser.error( f'when parsing {string}' '\nThe output file name must be .tab, .csv or .nc...') diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index f75ae4fd..feac6003 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -1100,6 +1100,8 @@ def run(self, params=None, return_columns=None, return_timestamps=None, If True, once the output dataframe has been formatted will split the xarrays in new columns following Vensim's naming to make a totally flat output. Default is True. + This argument will be ignored when passing a netCDF4 file + path in the output_file argument. cache_output: bool (optional) If True, the number of calls of outputs variables will be increased diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index 42b7db65..5bf971e8 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -157,9 +157,9 @@ def initialize(self, model, capture_elements): self.ds.description = "Results for simulation run on" \ f"{t.ctime(t.time())} using PySD version {__version__}" self.ds.model_file = model.py_model_file - self.ds.timestep = f"{model.components.time_step()}" - self.ds.initial_time = f"{model.components.initial_time()}" - self.ds.final_time = f"{model.components.final_time()}" + self.ds.timestep = f"{model.time.time_step()}" + self.ds.initial_time = f"{model.time.initial_time()}" + self.ds.final_time = f"{model.time.final_time()}" # creating variables for all model dimensions for dim_name, coords in model.subscripts.items(): @@ -241,9 +241,6 @@ def postprocess(self, **kwargs): # close Dataset self.ds.close() - if kwargs.get("flatten"): - warnings.warn("DataArrays stored in netCDF4 will not be flattened") - print(f"Results stored in {self.out_file}") def add_run_elements(self, model, capture_elements): @@ -444,8 +441,8 @@ def make_flat_df(df, return_addresses, flatten=False): Dataframe to process. return_addresses: dict - Keys will be column names of the resulting dataframe, and are what the - user passed in as 'return_columns'. Values are a tuple: + Keys will be column names of the resulting dataframe, and are what + the user passed in as 'return_columns'. Values are a tuple: (py_name, {coords dictionary}) which tells us where to look for the value to put in that specific column. diff --git a/tests/more-tests/numeric_subscripts/test_subscript_1d_arrays.mdl b/tests/more-tests/numeric_subscripts/test_subscript_1d_arrays.mdl new file mode 100644 index 00000000..efd1fa7c --- /dev/null +++ b/tests/more-tests/numeric_subscripts/test_subscript_1d_arrays.mdl @@ -0,0 +1,93 @@ +{UTF-8} +Inflow A[One Dimensional Subscript]= + Rate A[One Dimensional Subscript] + ~ + ~ | + +Stock A[One Dimensional Subscript]= INTEG ( + Inflow A[One Dimensional Subscript], + 0) + ~ + ~ | + +One Dimensional Subscript: + 1, 20, 300 + ~ + ~ | + +Rate A[One Dimensional Subscript]= + 0.01, 0.02, 0.03 + ~ + ~ | + +******************************************************** + .Control +********************************************************~ + Simulation Control Parameters + | + +FINAL TIME = 100 + ~ Month + ~ The final time for the simulation. + | + +INITIAL TIME = 0 + ~ Month + ~ The initial time for the simulation. + | + +SAVEPER = + TIME STEP + ~ Month [0,?] + ~ The frequency with which output is stored. + | + +TIME STEP = 1 + ~ Month [0,?] + ~ The time step for the simulation. + | + +\\\---/// Sketch information - do not modify anything except names +V300 Do not put anything below this section - it will be ignored +*View 1 +$192-192-192,0,Times New Roman|12||0-0-0|0-0-0|0-0-255|-1--1--1|-1--1--1|72,72,100,0 +10,1,Stock A,420,222,40,20,3,3,0,0,0,0,0,0 +12,2,48,257,223,10,8,0,3,0,0,-1,0,0,0 +1,3,5,1,4,0,0,22,0,0,0,-1--1--1,,1|(354,223)| +1,4,5,2,100,0,0,22,0,0,0,-1--1--1,,1|(292,223)| +11,5,48,323,223,6,8,34,3,0,0,1,0,0,0 +10,6,Inflow A,323,239,24,8,40,3,0,0,-1,0,0,0 +10,7,Rate A,318,308,19,8,8,3,0,0,0,0,0,0 +1,8,7,6,0,0,0,0,0,64,0,-1--1--1,,1|(319,280)| +///---\\\ +:L<%^E!@ +1:Current.vdf +9:Current +22:$,Dollar,Dollars,$s +22:Hour,Hours +22:Month,Months +22:Person,People,Persons +22:Unit,Units +22:Week,Weeks +22:Year,Years +22:Day,Days +15:0,0,0,0,0,0 +19:100,0 +27:2, +34:0, +4:Time +5:Stock A[One Dimensional Subscript] +35:Date +36:YYYY-MM-DD +37:2000 +38:1 +39:1 +40:2 +41:0 +42:1 +24:0 +25:100 +26:100 +6:1 +6:20 +6:300 diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index 1c8cf91a..573fb0dc 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -34,6 +34,9 @@ test_model_constant_pipe = more_tests.joinpath( "constant_pipeline/test_constant_pipeline.mdl") +test_model_numeric_coords = more_tests.joinpath( + "numeric_subscripts/test_subscript_1d_arrays.mdl" +) class TestPySD(): @@ -1806,9 +1809,28 @@ def test_output_with_dimensions(self, shared_tmpdir): simplefilter("always") model2.run(output_file=out_file2) - with nc.Dataset(out_file2, "r")as ds: + with nc.Dataset(out_file2, "r") as ds: assert "constant" in list(ds.variables.keys()) assert ds["constant"].dimensions == ("dim1",) + assert ds["dim1"][:].data.dtype == "S1" + + # dimension with numeric coords + model3 = pysd.read_vensim(test_model_numeric_coords) + model3.progress = False + + out_file3 = shared_tmpdir.joinpath("results3.nc") + + with catch_warnings(record=True) as w: + simplefilter("always") + model3.run(output_file=out_file3) + + with nc.Dataset(out_file3, "r")as ds: + assert np.array_equal( + ds["time"][:].data, np.arange(0.0, 101.0, 1.0)) + # coordinates get dtype=object when their length is different + assert ds["One Dimensional Subscript"][:].dtype == "O" + assert np.allclose( + ds["stock_a"][-1, :].data, np.array([1.0, 2.0, 3.0])) def test_make_flat_df(self): @@ -1838,7 +1860,8 @@ def test_make_flat_df_0dxarray(self): return_addresses = {'Elem1': ('elem1', {})} - actual = DataFrameHandler.make_flat_df(df, return_addresses, flatten=True) + actual = DataFrameHandler.make_flat_df( + df, return_addresses, flatten=True) # check all columns are in the DataFrame assert set(actual.columns) == set(expected.columns) @@ -1934,7 +1957,8 @@ def test_make_flat_df_flatten(self): 'Dim2': ['D', 'E', 'F']}), 'Elem2': ('elem2', {})} - actual = DataFrameHandler.make_flat_df(df, return_addresses, flatten=True) + actual = DataFrameHandler.make_flat_df( + df, return_addresses, flatten=True) # check all columns are in the DataFrame assert set(actual.columns) == set(expected.columns) @@ -1969,7 +1993,8 @@ def test_make_flat_df_flatten_transposed(self): return_addresses = { 'Elem2': ('elem2', {})} - actual = DataFrameHandler.make_flat_df(df, return_addresses, flatten=True) + actual = DataFrameHandler.make_flat_df( + df, return_addresses, flatten=True) # check all columns are in the DataFrame assert set(actual.columns) == set(expected.columns) From 78cb77256133cb28b2a6dd73563ab52887abff0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Tue, 6 Sep 2022 22:36:19 +0200 Subject: [PATCH 18/43] add all doc attrs to vars in nc --- pysd/py_backend/output.py | 9 +++------ tests/pytest_pysd/pytest_pysd.py | 5 +++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index 5bf971e8..f2bdf7f4 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -300,12 +300,9 @@ def __create_ds_vars(self, model, capture_elements, add_time=True): self.ds.createVariable(key, "f8", dims, compression="zlib") # adding units and description as metadata for each var - self.ds[key].units = model.doc.loc[ - model.doc["Py Name"] == key, - "Units"].values[0] or "Missing" - self.ds[key].description = model.doc.loc[ - model.doc["Py Name"] == key, - "Comment"].values[0] or "Missing" + for col in model.doc.columns: + setattr(self.ds[key], col, model.doc.loc[ + model.doc["Py Name"] == key, col].values[0] or "Missing") class DataFrameHandler(OutputHandlerInterface): diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index 573fb0dc..5f7dd5d0 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -1796,8 +1796,8 @@ def test_output_with_dimensions(self, shared_tmpdir): assert ds["lookup_1d_time"].dimensions == ("time",) assert ds["d2d"].dimensions == ("time", "Rows", "Dim") - assert ds["d2d"].description == "Missing" - assert ds["d2d"].units == "Missing" + assert ds["d2d"].Comment == "Missing" + assert ds["d2d"].Units == "Missing" # test cache run variables with dimensions model2 = pysd.read_vensim(test_model_constants) @@ -1831,6 +1831,7 @@ def test_output_with_dimensions(self, shared_tmpdir): assert ds["One Dimensional Subscript"][:].dtype == "O" assert np.allclose( ds["stock_a"][-1, :].data, np.array([1.0, 2.0, 3.0])) + assert set(model.doc.columns) == set(ds["stock_a"].ncattrs()) def test_make_flat_df(self): From e5d6f29998b5c6316141ac999420ce1cff7f859b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Fri, 9 Sep 2022 11:03:32 +0200 Subject: [PATCH 19/43] add test for property --- pysd/py_backend/output.py | 78 ++++++++++++++++---------------- tests/pytest_pysd/pytest_pysd.py | 15 ++++++ 2 files changed, 53 insertions(+), 40 deletions(-) diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index f2bdf7f4..20130244 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -131,8 +131,15 @@ class DatasetHandler(OutputHandlerInterface): def __init__(self, out_file): self.out_file = out_file - self.step = 0 self.ds = None + self._step = 0 + + @property + def step(self): + return self._step + + def __update_step(self): + self._step = self.step + 1 def initialize(self, model, capture_elements): """ @@ -154,9 +161,9 @@ def initialize(self, model, capture_elements): self.ds = nc.Dataset(self.out_file, "w") # defining global attributes - self.ds.description = "Results for simulation run on" \ + self.ds.description = "Results for simulation run on " \ f"{t.ctime(t.time())} using PySD version {__version__}" - self.ds.model_file = model.py_model_file + self.ds.model_file = model.py_model_file or model.mdl_file self.ds.timestep = f"{model.time.time_step()}" self.ds.initial_time = f"{model.time.initial_time()}" self.ds.final_time = f"{model.time.final_time()}" @@ -166,20 +173,21 @@ def initialize(self, model, capture_elements): coords = np.array(coords) # create dimension self.ds.createDimension(dim_name, len(coords)) + # length of the longest string in the coords max_str_len = len(max(coords, key=len)) - # create variable - # TODO: check if the type could be defined otherwise) + # create variable for the dimension var = self.ds.createVariable( dim_name, f"S{max_str_len}", (dim_name,)) - # assigning values to variable + + # assigning coords to dimension var[:] = coords # creating the time dimension as unlimited self.ds.createDimension("time", None) - # creating variables in capture_elements + # creating variables self.__create_ds_vars(model, capture_elements) def update(self, model, capture_elements): @@ -199,27 +207,19 @@ def update(self, model, capture_elements): """ for key in capture_elements: - comp = getattr(model.components, key) - comp_vals = comp() + comp = model[key] if "time" in self.ds[key].dimensions: - if isinstance(comp_vals, xr.DataArray): - self.ds[key][self.step, :] = comp_vals.values - elif isinstance(comp_vals, np.ndarray): - self.ds[key][self.step, :] = comp_vals + if isinstance(comp, xr.DataArray): + self.ds[key][self.step, :] = comp.values else: - self.ds[key][self.step] = comp_vals + self.ds[key][self.step] = comp else: try: # this issue can arise with external objects - if isinstance(comp_vals, xr.DataArray): - self.ds[key][:] = comp_vals.values - elif isinstance(comp_vals, np.ndarray): - if comp_vals.size == 1: - self.ds[key][:] = comp_vals - else: - self.ds[key][:] = comp_vals + if isinstance(comp, xr.DataArray): + self.ds[key][:] = comp.values else: - self.ds[key][:] = comp_vals + self.ds[key][:] = comp except ValueError: warnings.warn( f"The dimensions of {key} in the results " @@ -227,7 +227,7 @@ def update(self, model, capture_elements): "variable. The resulting values will not be " "included in the results file.") - self.step += 1 + self.__update_step() def postprocess(self, **kwargs): """ @@ -259,13 +259,11 @@ def add_run_elements(self, model, capture_elements): None """ # creating variables in capture_elements - # TODO we are looping through all capture elements twice. This - # could be avoided - self.__create_ds_vars(model, capture_elements, add_time=False) + self.__create_ds_vars(model, capture_elements, time_dim=False) self.update(model, capture_elements) - def __create_ds_vars(self, model, capture_elements, add_time=True): + def __create_ds_vars(self, model, capture_elements, time_dim=True): """ Create new variables in a netCDF4 Dataset from the capture_elements. @@ -275,34 +273,34 @@ def __create_ds_vars(self, model, capture_elements, add_time=True): PySD Model object capture_elements: set Which model elements to capture - uses pysafe names. - add_time: bool - Whether to add a time as the first dimension for the variables. + time_dim: bool + Whether to add time as the first dimension for the variable. Returns ------- None """ - for key in capture_elements: - comp = getattr(model.components, key) - comp_vals = comp() + comp = model[key] dims = () - if isinstance(comp_vals, (xr.DataArray, np.ndarray)): - if comp.subscripts: - dims = tuple(comp.subscripts) + if isinstance(comp, xr.DataArray): + dims = tuple(comp.dims) - if add_time: + if time_dim: dims = ("time",) + dims - self.ds.createVariable(key, "f8", dims, compression="zlib") + var = self.ds.createVariable(key, "f8", dims, compression="zlib") - # adding units and description as metadata for each var + # adding metadata for each var from the model.doc for col in model.doc.columns: - setattr(self.ds[key], col, model.doc.loc[ - model.doc["Py Name"] == key, col].values[0] or "Missing") + var.setncattr( + col, + model.doc.loc[model.doc["Py Name"] == key, col].values[0] \ + or "Missing" + ) class DataFrameHandler(OutputHandlerInterface): diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index 5f7dd5d0..bc78a317 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -1833,6 +1833,21 @@ def test_output_with_dimensions(self, shared_tmpdir): ds["stock_a"][-1, :].data, np.array([1.0, 2.0, 3.0])) assert set(model.doc.columns) == set(ds["stock_a"].ncattrs()) + def test_dataset_handler_step_setter(self, shared_tmpdir): + model = pysd.read_vensim(test_model_look) + capture_elements = set() + results = shared_tmpdir.joinpath("results.nc") + output = ModelOutput(model, capture_elements, results) + + # Dataset handler step cannot be modified from the outside + with pytest.raises(AttributeError): + output.handler.step = 5 + + with pytest.raises(AttributeError): + output.handler.__update_step() + + assert output.handler.step == 0 + def test_make_flat_df(self): df = pd.DataFrame(index=[1], columns=['elem1']) From 0893709a9fd0b8d7b8f370bc44f61e1d75475f9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Fri, 9 Sep 2022 11:49:29 +0200 Subject: [PATCH 20/43] add test for variable timestep and final_time --- pysd/py_backend/output.py | 6 ++++-- tests/pytest_pysd/pytest_pysd.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index 20130244..75802518 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -164,9 +164,11 @@ def initialize(self, model, capture_elements): self.ds.description = "Results for simulation run on " \ f"{t.ctime(t.time())} using PySD version {__version__}" self.ds.model_file = model.py_model_file or model.mdl_file - self.ds.timestep = f"{model.time.time_step()}" + self.ds.timestep = f"{model.time.time_step()}" if model.cache_type[ + "time_step"] == "run" else "Variable" self.ds.initial_time = f"{model.time.initial_time()}" - self.ds.final_time = f"{model.time.final_time()}" + self.ds.final_time = f"{model.time.final_time()}" if model.cache_type[ + "final_time"] == "run" else "Variable" # creating variables for all model dimensions for dim_name, coords in model.subscripts.items(): diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index bc78a317..ac4dc0b1 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -30,6 +30,10 @@ "test-models/tests/get_constants_subranges/" "test_get_constants_subranges.mdl" ) +test_variable_step = _root.joinpath( + "test-models/tests/control_vars/" + "test_control_vars.mdl" +) more_tests = _root.joinpath("more-tests/") test_model_constant_pipe = more_tests.joinpath( @@ -1833,6 +1837,15 @@ def test_output_with_dimensions(self, shared_tmpdir): ds["stock_a"][-1, :].data, np.array([1.0, 2.0, 3.0])) assert set(model.doc.columns) == set(ds["stock_a"].ncattrs()) + def test_variable_time_step(self, shared_tmpdir): + model = pysd.read_vensim(test_variable_step) + capture_elements = set() + results = shared_tmpdir.joinpath("results.nc") + output = ModelOutput(model, capture_elements, results) + dataset = output.handler.ds + assert dataset.timestep == "Variable" + assert dataset.final_time == "Variable" + def test_dataset_handler_step_setter(self, shared_tmpdir): model = pysd.read_vensim(test_model_look) capture_elements = set() From 6111e8989a446f7a1e2f410b0453ef1a85bb4c92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Fri, 9 Sep 2022 12:17:53 +0200 Subject: [PATCH 21/43] fix pep8 --- pysd/cli/parser.py | 5 +++-- pysd/py_backend/output.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pysd/cli/parser.py b/pysd/cli/parser.py index 707d91f4..0f1f5d8b 100644 --- a/pysd/cli/parser.py +++ b/pysd/cli/parser.py @@ -29,8 +29,9 @@ def check_output(string): Checks that out put file ends with .tab or .csv """ - if not string.endswith('.tab') and not string.endswith('.csv') and not \ - string.endswith('.nc'): + if (not string.endswith('.tab') and + not string.endswith('.csv') and + not string.endswith('.nc')): parser.error( f'when parsing {string}' '\nThe output file name must be .tab, .csv or .nc...') diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index 75802518..b3506864 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -300,8 +300,8 @@ def __create_ds_vars(self, model, capture_elements, time_dim=True): for col in model.doc.columns: var.setncattr( col, - model.doc.loc[model.doc["Py Name"] == key, col].values[0] \ - or "Missing" + model.doc.loc[model.doc["Py Name"] == key, col].values[0] + or "Missing" ) From c1bc127e99fc20a05483d6add72bd50f7894b11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Fri, 9 Sep 2022 12:39:29 +0200 Subject: [PATCH 22/43] update whats new --- docs/whats_new.rst | 9 +++++++-- pysd/cli/parser.py | 4 +--- pysd/py_backend/model.py | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/whats_new.rst b/docs/whats_new.rst index 9a46f396..1ea3764a 100644 --- a/docs/whats_new.rst +++ b/docs/whats_new.rst @@ -5,6 +5,8 @@ v3.7.0 (to be released) New Features ~~~~~~~~~~~~ +- Simulation results can now be stored as netCDF4 files (:issue:`355`). (`@rogersamso `_) +- The CLI also accepts netCDF4 file paths after the -o argument. (`@rogersamso `_) Breaking changes ~~~~~~~~~~~~~~~~ @@ -14,17 +16,20 @@ Deprecations Bug fixes ~~~~~~~~~ -- Fix bug when a WITH LOOKUPS argument has subscripts. +- Fix bug when a WITH LOOKUPS argument has subscripts. (`@enekomartinmartinez `_) Documentation ~~~~~~~~~~~~~ +- Adds Storing simulation results on a file section in the getting started page. (`@rogersamso `_) Performance ~~~~~~~~~~~ +- Exporting outputs as netCDF4 is much faster than exporting a pandas DataFrame, especially for large models. (`@rogersamso `_) Internal Changes ~~~~~~~~~~~~~~~~ -- Make PySD work with :py:mod:`parsimonius` 0.10.0. +- Make PySD work with :py:mod:`parsimonius` 0.10.0. (`@enekomartinmartinez `_) +- Add netCDF4 and hdf5 dependencies. (`@rogersamso `_) v3.6.1 (2022/09/05) diff --git a/pysd/cli/parser.py b/pysd/cli/parser.py index 0f1f5d8b..b22bb677 100644 --- a/pysd/cli/parser.py +++ b/pysd/cli/parser.py @@ -29,9 +29,7 @@ def check_output(string): Checks that out put file ends with .tab or .csv """ - if (not string.endswith('.tab') and - not string.endswith('.csv') and - not string.endswith('.nc')): + if not string.endswith(('.tab', '.csv', '.nc')): parser.error( f'when parsing {string}' '\nThe output file name must be .tab, .csv or .nc...') diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index feac6003..47392e58 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -712,10 +712,10 @@ def set_components(self, params, new=False): >>> model.set_components({'birth_rate': br}) """ - # TODO: allow the params argument to take a pandas dataframe, where + # TODO: allow the params argument to take a pandas dataframe, where # column names are variable names. However some variables may be # constant or have no values for some index. This should be processed. - # TODO: make this compatible with loading outputs from other files + # TODO: make this compatible with loading outputs from other files for key, value in params.items(): func_name = utils.get_key_and_value_by_insensitive_key_or_value( From 4efe997d500b5b7c30195df3d7252a1c45cea114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Fri, 9 Sep 2022 14:02:11 +0200 Subject: [PATCH 23/43] fix docstring --- pysd/py_backend/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index 47392e58..c4f10b3f 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -1112,7 +1112,7 @@ def run(self, params=None, return_columns=None, return_timestamps=None, output_file: str, pathlib.Path or None (optional) Path of the file in which to save simulation results. - For now, only netCDF4 files (.nc) are supported. + Currently, csv, tab and nc (netCDF4) files are supported. Examples From ed59742f35fbef16042f6e620ab852864fbc03c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Fri, 9 Sep 2022 15:08:41 +0200 Subject: [PATCH 24/43] ci.yml upgrade setuptools --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d51229e..23f580dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | + pip install --upgrade pip setuptools pip install -r tests/requirements.txt pip install -e . - name: Test and coverage From 926f4eb8b237a294ed478a69b54bc396dce8f753 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez <61285767+enekomartinmartinez@users.noreply.github.com> Date: Fri, 9 Sep 2022 16:59:46 +0200 Subject: [PATCH 25/43] Update requirements.txt --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1661adbf..7f50eb21 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,7 @@ pandas parsimonious xarray -h5py -netCDF4 +netCDF4<=1.5 xlrd lxml regex From c9dd3f159ca4f690b4545031cabbb0e79a13c6f3 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez <61285767+enekomartinmartinez@users.noreply.github.com> Date: Fri, 9 Sep 2022 17:01:57 +0200 Subject: [PATCH 26/43] Update requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7f50eb21..aeca820c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ pandas parsimonious xarray +h5py netCDF4<=1.5 xlrd lxml From 4f4af58d749a220a979ab032e30c0626aeee45cb Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez <61285767+enekomartinmartinez@users.noreply.github.com> Date: Fri, 9 Sep 2022 17:06:07 +0200 Subject: [PATCH 27/43] Update requirements.txt --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index aeca820c..7199ecef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ pandas parsimonious xarray -h5py -netCDF4<=1.5 +netCDF4==1.5; sys_platform == 'win' +netCDF4==1.6; sys_platform != 'win' xlrd lxml regex From 14539d0eeda74cdf0b7d5ed3fd374e163e8e3d76 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez <61285767+enekomartinmartinez@users.noreply.github.com> Date: Fri, 9 Sep 2022 17:11:38 +0200 Subject: [PATCH 28/43] Update requirements.txt --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7199ecef..d03c2a22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ pandas parsimonious xarray -netCDF4==1.5; sys_platform == 'win' -netCDF4==1.6; sys_platform != 'win' +netCDF4==1.5; sys_platform == 'win32' and python_version == "3.7" +netCDF4==1.6; sys_platform != 'win32' or python_version != "3.7" xlrd lxml regex From 6f7a9bbc57383022836df63a6e5becd5c30822ed Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez <61285767+enekomartinmartinez@users.noreply.github.com> Date: Fri, 9 Sep 2022 17:13:50 +0200 Subject: [PATCH 29/43] Update requirements.txt --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d03c2a22..cf9a1aca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ pandas parsimonious xarray -netCDF4==1.5; sys_platform == 'win32' and python_version == "3.7" -netCDF4==1.6; sys_platform != 'win32' or python_version != "3.7" +netCDF4==1.5; platform_system == 'Windows' and python_version == "3.7" +netCDF4==1.6; platform_system != 'Windows' or python_version != "3.7" xlrd lxml regex From dc69b1e559a0f78d9885e33e0fb23bb07610559a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Sat, 10 Sep 2022 15:30:48 +0200 Subject: [PATCH 30/43] remove test from more-tests --- .../test_subscript_1d_arrays.mdl | 93 ------------------- tests/pytest_pysd/pytest_pysd.py | 9 +- 2 files changed, 6 insertions(+), 96 deletions(-) delete mode 100644 tests/more-tests/numeric_subscripts/test_subscript_1d_arrays.mdl diff --git a/tests/more-tests/numeric_subscripts/test_subscript_1d_arrays.mdl b/tests/more-tests/numeric_subscripts/test_subscript_1d_arrays.mdl deleted file mode 100644 index efd1fa7c..00000000 --- a/tests/more-tests/numeric_subscripts/test_subscript_1d_arrays.mdl +++ /dev/null @@ -1,93 +0,0 @@ -{UTF-8} -Inflow A[One Dimensional Subscript]= - Rate A[One Dimensional Subscript] - ~ - ~ | - -Stock A[One Dimensional Subscript]= INTEG ( - Inflow A[One Dimensional Subscript], - 0) - ~ - ~ | - -One Dimensional Subscript: - 1, 20, 300 - ~ - ~ | - -Rate A[One Dimensional Subscript]= - 0.01, 0.02, 0.03 - ~ - ~ | - -******************************************************** - .Control -********************************************************~ - Simulation Control Parameters - | - -FINAL TIME = 100 - ~ Month - ~ The final time for the simulation. - | - -INITIAL TIME = 0 - ~ Month - ~ The initial time for the simulation. - | - -SAVEPER = - TIME STEP - ~ Month [0,?] - ~ The frequency with which output is stored. - | - -TIME STEP = 1 - ~ Month [0,?] - ~ The time step for the simulation. - | - -\\\---/// Sketch information - do not modify anything except names -V300 Do not put anything below this section - it will be ignored -*View 1 -$192-192-192,0,Times New Roman|12||0-0-0|0-0-0|0-0-255|-1--1--1|-1--1--1|72,72,100,0 -10,1,Stock A,420,222,40,20,3,3,0,0,0,0,0,0 -12,2,48,257,223,10,8,0,3,0,0,-1,0,0,0 -1,3,5,1,4,0,0,22,0,0,0,-1--1--1,,1|(354,223)| -1,4,5,2,100,0,0,22,0,0,0,-1--1--1,,1|(292,223)| -11,5,48,323,223,6,8,34,3,0,0,1,0,0,0 -10,6,Inflow A,323,239,24,8,40,3,0,0,-1,0,0,0 -10,7,Rate A,318,308,19,8,8,3,0,0,0,0,0,0 -1,8,7,6,0,0,0,0,0,64,0,-1--1--1,,1|(319,280)| -///---\\\ -:L<%^E!@ -1:Current.vdf -9:Current -22:$,Dollar,Dollars,$s -22:Hour,Hours -22:Month,Months -22:Person,People,Persons -22:Unit,Units -22:Week,Weeks -22:Year,Years -22:Day,Days -15:0,0,0,0,0,0 -19:100,0 -27:2, -34:0, -4:Time -5:Stock A[One Dimensional Subscript] -35:Date -36:YYYY-MM-DD -37:2000 -38:1 -39:1 -40:2 -41:0 -42:1 -24:0 -25:100 -26:100 -6:1 -6:20 -6:300 diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index ac4dc0b1..f9381207 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -34,13 +34,16 @@ "test-models/tests/control_vars/" "test_control_vars.mdl" ) +test_model_numeric_coords = _root.joinpath( + "test-models/tests/subscript_1d_arrays/" + "test_subscript_1d_arrays.mdl" +) + more_tests = _root.joinpath("more-tests/") test_model_constant_pipe = more_tests.joinpath( "constant_pipeline/test_constant_pipeline.mdl") -test_model_numeric_coords = more_tests.joinpath( - "numeric_subscripts/test_subscript_1d_arrays.mdl" -) + class TestPySD(): From 5cc8a727454986b2372a934fe441fcece95862f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Sat, 10 Sep 2022 15:59:46 +0200 Subject: [PATCH 31/43] conditional compression --- pysd/py_backend/output.py | 9 ++++++++- pysd/pysd.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index b3506864..77485d47 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -268,6 +268,7 @@ def add_run_elements(self, model, capture_elements): def __create_ds_vars(self, model, capture_elements, time_dim=True): """ Create new variables in a netCDF4 Dataset from the capture_elements. + Data is zlib compressed by default for netCDF4 1.6.0 and above. Parameters ---------- @@ -283,6 +284,12 @@ def __create_ds_vars(self, model, capture_elements, time_dim=True): None """ + if tuple(nc.__version__.split(".")) >= ('1', '6', '0'): + var_ = lambda key, dims: self.ds.createVariable( + key, "f8", dims, compression="zlib") + else: + var_ = lambda key, dims: self.ds.createVariable(key, "f8", dims) + for key in capture_elements: comp = model[key] @@ -294,7 +301,7 @@ def __create_ds_vars(self, model, capture_elements, time_dim=True): if time_dim: dims = ("time",) + dims - var = self.ds.createVariable(key, "f8", dims, compression="zlib") + var = var_(key, dims) # adding metadata for each var from the model.doc for col in model.doc.columns: diff --git a/pysd/pysd.py b/pysd/pysd.py index 21b6f93d..09bcdd64 100644 --- a/pysd/pysd.py +++ b/pysd/pysd.py @@ -13,7 +13,7 @@ if sys.version_info[:2] < (3, 7): # pragma: no cover raise RuntimeError( "\n\n" - + "Your Python version is not longer supported by PySD.\n" + + "Your Python version is no longer supported by PySD.\n" + "The current version needs to run at least Python 3.7." + " You are running:\n\tPython " + sys.version From ec09d1e36f350e1525c31b48cd8bcf8586e3b4eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Sun, 11 Sep 2022 17:43:38 +0200 Subject: [PATCH 32/43] working implementation --- pysd/py_backend/output.py | 81 +++++++++++++++++++++++++++++++-------- 1 file changed, 66 insertions(+), 15 deletions(-) diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index 77485d47..082c3fdc 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -41,22 +41,15 @@ class ModelOutput(): def __init__(self, model, capture_elements, out_file=None): - self.handler = self.__handle(out_file) + # Add any other handlers that you write here, in the order you + # want them to run (DataFrameHandler runs first) + self.handler = DataFrameHandler(DatasetHandler(None)).process_output(out_file) capture_elements.add("time") self.capture_elements = capture_elements self.initialize(model) - def __handle(self, out_file): - # TODO improve the handler to avoid if then else statements - if out_file: - if out_file.suffix == ".nc": - return DatasetHandler(out_file) - # when the users expects a csv or tab output file, it defaults to the - # DataFrame path - return DataFrameHandler(out_file) - def initialize(self, model): """ Delegating the creation of the results object and its elements to the appropriate handler.""" @@ -82,9 +75,14 @@ class OutputHandlerInterface(metaclass=abc.ABCMeta): """ Interface for the creation of different output type handlers. """ + def __init__(self, next=None): + self._next = next + @classmethod def __subclasshook__(cls, subclass): - return (hasattr(subclass, 'initialize') and + return (hasattr(subclass, 'process_output') and + callable(subclass.process_output) and + hasattr(subclass, 'initialize') and callable(subclass.initialize) and hasattr(subclass, 'update') and callable(subclass.update) and @@ -94,6 +92,13 @@ def __subclasshook__(cls, subclass): callable(subclass.add_run_elements) or NotImplemented) + @abc.abstractmethod + def process_output(self, out_file): + """ + If concrete handler can process out_file, returns True, else False. + """ + raise NotImplementedError + @abc.abstractmethod def initialize(self, model, capture_elements): """ @@ -129,8 +134,9 @@ class DatasetHandler(OutputHandlerInterface): Manages simulation results stored as netCDF4 Dataset. """ - def __init__(self, out_file): - self.out_file = out_file + def __init__(self, next): + super().__init__(next) + self.out_file = None self.ds = None self._step = 0 @@ -141,6 +147,28 @@ def step(self): def __update_step(self): self._step = self.step + 1 + def process_output(self, out_file): + """ + If out_file can be handled by this concrete handler it returns True, + else False. + + Parameters + ---------- + out_file: str or pathlib.Path + Path to the file where the results will be written. + + Returns + ------- + bool + + """ + if out_file: + if out_file.suffix == ".nc": + self.out_file = out_file + return self + elif self._next is not None: + return self._next.process_output(out_file) + def initialize(self, model, capture_elements): """ Creates a netCDF4 Dataset and adds model dimensions and variables @@ -316,9 +344,32 @@ class DataFrameHandler(OutputHandlerInterface): """ Manages simulation results stored as pandas DataFrame. """ - def __init__(self, out_file): + def __init__(self, next): + super().__init__(next) self.ds = None - self.output_file = out_file + self.output_file = None + + def process_output(self, out_file): + """ + If this handler can process out_file, it returns True, else False. + + Parameters + ---------- + out_file: str or pathlib.Path + Path to the file where the results will be written. + + Returns + ------- + bool + + """ + self.out_file = out_file + if not out_file: + return self + if out_file.suffix in [".csv", ".tab"]: + return self + elif self._next is not None: + return self._next.process_output(out_file) def initialize(self, model, capture_elements): """ From 79be3d0e6899fc71d70f5510cdc23d875286f059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Sun, 11 Sep 2022 18:02:30 +0200 Subject: [PATCH 33/43] alternative implementation --- pysd/py_backend/output.py | 68 +++++++++++++++++++++++--------- tests/pytest_pysd/pytest_pysd.py | 3 +- 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index 082c3fdc..f4217a73 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -43,7 +43,7 @@ def __init__(self, model, capture_elements, out_file=None): # Add any other handlers that you write here, in the order you # want them to run (DataFrameHandler runs first) - self.handler = DataFrameHandler(DatasetHandler(None)).process_output(out_file) + self.handler = DataFrameHandler(DatasetHandler(None)).handle(out_file) capture_elements.add("time") self.capture_elements = capture_elements @@ -73,11 +73,32 @@ def add_run_elements(self, model, run_elements): class OutputHandlerInterface(metaclass=abc.ABCMeta): """ - Interface for the creation of different output type handlers. + Interface for the creation of different output handlers. """ def __init__(self, next=None): self._next = next + def handle(self, out_file): + """ + If the concrete handler can write on the output file type passed by the + user, it returns the handler itself, else it goes to the next handler. + + Parameters + ---------- + out_file: str or pathlib.Path + Path to the file where the results will be written. + + Returns + ------- + handler + """ + handler = self.process_output(out_file) + + if handler is not None: # the handler can write the out_file type. + return handler + else: + return self._next.handle(out_file) + @classmethod def __subclasshook__(cls, subclass): return (hasattr(subclass, 'process_output') and @@ -95,7 +116,8 @@ def __subclasshook__(cls, subclass): @abc.abstractmethod def process_output(self, out_file): """ - If concrete handler can process out_file, returns True, else False. + If concrete handler can process out_file, returns it, else returns + None. """ raise NotImplementedError @@ -142,15 +164,22 @@ def __init__(self, next): @property def step(self): + """ + Used as time index for the output Dataset. Increases by one at each + iteration. + """ return self._step def __update_step(self): + """ + Increases the _step attribute by 1 at each model iteration. + """ self._step = self.step + 1 def process_output(self, out_file): """ - If out_file can be handled by this concrete handler it returns True, - else False. + If out_file can be handled by this concrete handler, it returns the + handler instance, else it returns None. Parameters ---------- @@ -159,15 +188,14 @@ def process_output(self, out_file): Returns ------- - bool + None or DatasetHandler instance """ if out_file: if out_file.suffix == ".nc": self.out_file = out_file return self - elif self._next is not None: - return self._next.process_output(out_file) + return None def initialize(self, model, capture_elements): """ @@ -267,8 +295,6 @@ def postprocess(self, **kwargs): ------- None """ - - # close Dataset self.ds.close() print(f"Results stored in {self.out_file}") @@ -347,11 +373,13 @@ class DataFrameHandler(OutputHandlerInterface): def __init__(self, next): super().__init__(next) self.ds = None - self.output_file = None + self.out_file = None def process_output(self, out_file): """ If this handler can process out_file, it returns True, else False. + DataFrameHandler handles outputs to be saved as *.csv or *.tab files, + and is the default handler when no output file is passed by the user. Parameters ---------- @@ -360,16 +388,18 @@ def process_output(self, out_file): Returns ------- - bool + None or DataFrameHandler instance """ self.out_file = out_file + if not out_file: return self + if out_file.suffix in [".csv", ".tab"]: return self - elif self._next is not None: - return self._next.process_output(out_file) + + return None def initialize(self, model, capture_elements): """ @@ -425,12 +455,12 @@ def postprocess(self, **kwargs): del self.ds["time"] # enforce flattening if df is to be saved to csv or tab file - flatten = True if self.output_file else kwargs.get("flatten", None) + flatten = True if self.out_file else kwargs.get("flatten", None) df = DataFrameHandler.make_flat_df( self.ds, kwargs["return_addresses"], flatten ) - if self.output_file: + if self.out_file: self.__save_to_file(df) return df @@ -451,7 +481,7 @@ def __save_to_file(self, output): """ - if self.output_file.suffix == ".tab": + if self.out_file.suffix == ".tab": sep = "\t" else: sep = "," @@ -459,10 +489,10 @@ def __save_to_file(self, output): # QUOTE_NONE used to print the csv/tab files as vensim does with # special characterse, e.g.: "my-var"[Dimension] output.to_csv( - self.output_file, sep, index_label="Time", quoting=QUOTE_NONE + self.out_file, sep, index_label="Time", quoting=QUOTE_NONE ) - print(f"Data saved in '{self.output_file}'") + print(f"Data saved in '{self.out_file}'") def add_run_elements(self, model, capture_elements): """ diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index f9381207..978f8624 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -1704,13 +1704,14 @@ def test_output_handler_interface(self): # Add any additional Handler here. assert issubclass(DatasetHandler, OutputHandlerInterface) assert issubclass(DataFrameHandler, OutputHandlerInterface) - assert issubclass(ModelOutput, OutputHandlerInterface) class ThatFollowsInterface: """ This class does not inherit from OutputHandlerInterface, but it overrides all its methods (it follows the interface). """ + def process_output(self, out_file): + pass def initialize(self, model, capture_elements): pass From f8389b30fe3107e14afdda4a11c9bf7daa6567f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Sun, 11 Sep 2022 23:11:18 +0200 Subject: [PATCH 34/43] split update method for run and step elements --- pysd/py_backend/output.py | 50 +++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index f4217a73..77720979 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -250,7 +250,8 @@ def initialize(self, model, capture_elements): def update(self, model, capture_elements): """ - Writes values of variables in capture_elements in the netCDF4 Dataset. + Writes values of cache step variables from the capture_elements set + in the netCDF4 Dataset. Parameters ---------- @@ -267,26 +268,39 @@ def update(self, model, capture_elements): comp = model[key] - if "time" in self.ds[key].dimensions: - if isinstance(comp, xr.DataArray): - self.ds[key][self.step, :] = comp.values - else: - self.ds[key][self.step] = comp + if isinstance(comp, xr.DataArray): + self.ds[key][self.step, :] = comp.values else: - try: # this issue can arise with external objects - if isinstance(comp, xr.DataArray): - self.ds[key][:] = comp.values - else: - self.ds[key][:] = comp - except ValueError: - warnings.warn( - f"The dimensions of {key} in the results " - "do not match the declared dimensions for this " - "variable. The resulting values will not be " - "included in the results file.") + self.ds[key][self.step] = comp self.__update_step() + def __update_run_elements(self, model, capture_elements): + """ + Writes values of cache run elements from the cature_elements set + in the netCDF4 Dataset. + Cache run elements do not have the time dimension. + + Parameters + ---------- + model: pysd.Model + PySD Model object + capture_elements: set + Which model elements to capture - uses pysafe names. + + Returns + ------- + None + """ + for key in capture_elements: + + comp = model[key] + + if isinstance(comp, xr.DataArray): + self.ds[key][:] = comp.values + else: + self.ds[key][:] = comp + def postprocess(self, **kwargs): """ Closes netCDF4 Dataset. @@ -317,7 +331,7 @@ def add_run_elements(self, model, capture_elements): # creating variables in capture_elements self.__create_ds_vars(model, capture_elements, time_dim=False) - self.update(model, capture_elements) + self.__update_run_elements(model, capture_elements) def __create_ds_vars(self, model, capture_elements, time_dim=True): """ From db7569fc640c64fe00d92192175c8e6c90cdb069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Sun, 11 Sep 2022 23:50:20 +0200 Subject: [PATCH 35/43] better handling of compression --- pysd/py_backend/output.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index 77720979..a0ec5499 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -94,7 +94,7 @@ def handle(self, out_file): """ handler = self.process_output(out_file) - if handler is not None: # the handler can write the out_file type. + if handler is not None: # the handler can write the out_file type. return handler else: return self._next.handle(out_file) @@ -352,11 +352,10 @@ def __create_ds_vars(self, model, capture_elements, time_dim=True): None """ + kwargs = dict() + if tuple(nc.__version__.split(".")) >= ('1', '6', '0'): - var_ = lambda key, dims: self.ds.createVariable( - key, "f8", dims, compression="zlib") - else: - var_ = lambda key, dims: self.ds.createVariable(key, "f8", dims) + kwargs.update({"compression": "zlib"}) for key in capture_elements: comp = model[key] @@ -369,7 +368,7 @@ def __create_ds_vars(self, model, capture_elements, time_dim=True): if time_dim: dims = ("time",) + dims - var = var_(key, dims) + var = self.ds.createVariable(key, "f8", dims, **kwargs) # adding metadata for each var from the model.doc for col in model.doc.columns: From edf9617deda770724b0b3689311303835bc71f84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roger=20Sams=C3=B3?= Date: Mon, 12 Sep 2022 13:22:23 +0200 Subject: [PATCH 36/43] add tests, fix export to csv --- pysd/py_backend/output.py | 7 +- tests/pytest_pysd/pytest_output.py | 467 +++++++++++++++++++++++++++++ tests/pytest_pysd/pytest_pysd.py | 384 +----------------------- 3 files changed, 473 insertions(+), 385 deletions(-) create mode 100644 tests/pytest_pysd/pytest_output.py diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index a0ec5499..8b123611 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -195,7 +195,6 @@ def process_output(self, out_file): if out_file.suffix == ".nc": self.out_file = out_file return self - return None def initialize(self, model, capture_elements): """ @@ -412,8 +411,6 @@ def process_output(self, out_file): if out_file.suffix in [".csv", ".tab"]: return self - return None - def initialize(self, model, capture_elements): """ Creates a pandas DataFrame and adds model variables as columns. @@ -498,12 +495,12 @@ def __save_to_file(self, output): sep = "\t" else: sep = "," + output.columns = [col.replace(",", ";") for col in output.columns] # QUOTE_NONE used to print the csv/tab files as vensim does with # special characterse, e.g.: "my-var"[Dimension] output.to_csv( - self.out_file, sep, index_label="Time", quoting=QUOTE_NONE - ) + self.out_file, sep, index_label="Time", quoting=QUOTE_NONE) print(f"Data saved in '{self.out_file}'") diff --git a/tests/pytest_pysd/pytest_output.py b/tests/pytest_pysd/pytest_output.py new file mode 100644 index 00000000..b251f8be --- /dev/null +++ b/tests/pytest_pysd/pytest_output.py @@ -0,0 +1,467 @@ +from pathlib import Path +from warnings import simplefilter, catch_warnings + +import pytest +import numpy as np +import pandas as pd +import xarray as xr +import netCDF4 as nc + +from pysd.tools.benchmarking import assert_frames_close +from pysd.py_backend.output import OutputHandlerInterface, DatasetHandler, \ + DataFrameHandler, ModelOutput + +import pysd + + +_root = Path(__file__).parent.parent + +test_model_look = _root.joinpath( + "test-models/tests/get_lookups_subscripted_args/" + + "test_get_lookups_subscripted_args.mdl") +test_model_constants = _root.joinpath( + "test-models/tests/get_constants_subranges/" + "test_get_constants_subranges.mdl" +) +test_model_numeric_coords = _root.joinpath( + "test-models/tests/subscript_1d_arrays/" + "test_subscript_1d_arrays.mdl" +) +test_variable_step = _root.joinpath( + "test-models/tests/control_vars/" + "test_control_vars.mdl" +) + + +class TestOutput(): + + def test_output_handler_interface(self): + # when the class does not inherit from OutputHandlerInterface, it must + # implement all the interface to be a subclass of + # OutputHandlerInterface. + # Add any additional Handler here. + assert issubclass(DatasetHandler, OutputHandlerInterface) + assert issubclass(DataFrameHandler, OutputHandlerInterface) + + class ThatFollowsInterface: + """ + This class does not inherit from OutputHandlerInterface, but it + overrides all its methods (it follows the interface). + """ + def process_output(self, out_file): + pass + + def initialize(self, model, capture_elements): + pass + + def update(self, model, capture_elements): + pass + + def postprocess(self, **kwargs): + pass + + def add_run_elements(self, capture_elemetns): + pass + + # eventhough it does not inherit from OutputHandlerInterface, it is + # considered a subclass, because it follows the interface + assert issubclass(ThatFollowsInterface, OutputHandlerInterface) + + class IncompleteHandler: + """ + Class that does not follow the full interface + (add_run_elements is missing). + """ + def initialize(self, model, capture_elements): + pass + + def update(self, model, capture_elements): + pass + + def postprocess(self, **kwargs): + pass + + # It does not inherit from OutputHandlerInterface and does not fulfill + # its interface + assert not issubclass(IncompleteHandler, OutputHandlerInterface) + + class EmptyHandler(OutputHandlerInterface): + """ + When the class DOES inherit from OutputHandlerInterface, but does + not override all its abstract methods, then it cannot be + instantiated + """ + pass + + # it is a subclass because it inherits from it + assert issubclass(EmptyHandler, OutputHandlerInterface) + + # it cannot be instantiated because it does not override all abstract + # methods + with pytest.raises(TypeError): + empty = EmptyHandler() + + # calling methods that are not overriden returns NotImplementedError + # this should never happen, because these methods are instance methods, + # therefore the class needs to be instantiated first + with pytest.raises(NotImplementedError): + EmptyHandler.initialize(EmptyHandler, "model", "capture") + + with pytest.raises(NotImplementedError): + EmptyHandler.process_output(EmptyHandler, "out_file") + + with pytest.raises(NotImplementedError): + EmptyHandler.update(EmptyHandler, "model", "capture") + + with pytest.raises(NotImplementedError): + EmptyHandler.postprocess(EmptyHandler) + + with pytest.raises(NotImplementedError): + EmptyHandler.add_run_elements( + EmptyHandler, "model", "capture") + + def test_output_nc(self, shared_tmpdir): + model = pysd.read_vensim(test_model_look) + model.progress = False + + out_file = shared_tmpdir.joinpath("results.nc") + + with catch_warnings(record=True) as w: + simplefilter("always") + model.run(output_file=out_file) + + with nc.Dataset(out_file, "r") as ds: + assert ds.ncattrs() == [ + 'description', 'model_file', 'timestep', 'initial_time', + 'final_time'] + assert list(ds.dimensions.keys()) == ["Rows", "Dim", "time"] + # dimensions are stored as variables + assert ds["Rows"].size == 2 + assert "Rows" in ds.variables.keys() + assert "time" in ds.variables.keys() + # scalars do not have the time dimension + assert ds["initial_time"].size == 1 + # cache step variables have the "time" dimension + assert ds["lookup_1d_time"].dimensions == ("time",) + + assert ds["d2d"].dimensions == ("time", "Rows", "Dim") + + with catch_warnings(record=True) as w: + simplefilter("always") + assert ds["d2d"].Comment == "Missing" + assert ds["d2d"].Units == "Missing" + + # test cache run variables with dimensions + model2 = pysd.read_vensim(test_model_constants) + model2.progress = False + + out_file2 = shared_tmpdir.joinpath("results2.nc") + + with catch_warnings(record=True) as w: + simplefilter("always") + model2.run(output_file=out_file2) + + with nc.Dataset(out_file2, "r") as ds: + assert ds["time_step"].size == 1 + assert "constant" in list(ds.variables.keys()) + assert ds["constant"].dimensions == ("dim1",) + + with catch_warnings(record=True) as w: + simplefilter("always") + assert ds["dim1"][:].data.dtype == "S1" + + # dimension with numeric coords + model3 = pysd.read_vensim(test_model_numeric_coords) + model3.progress = False + + out_file3 = shared_tmpdir.joinpath("results3.nc") + + with catch_warnings(record=True) as w: + simplefilter("always") + model3.run(output_file=out_file3) + + # using xarray instead of netCDF4 to load the dataset + + with catch_warnings(record=True) as w: + simplefilter("always") + ds = xr.open_dataset(out_file3, engine="netcdf4") + + assert "time" in ds.dims + assert ds["rate_a"].dims == ("One Dimensional Subscript",) + assert ds["stock_a"].dims == ("time", "One Dimensional Subscript") + + # coordinates get dtype=object when their length is different + assert ds["One Dimensional Subscript"].data.dtype == "O" + + # check data + assert np.array_equal( + ds["time"].data, np.arange(0.0, 101.0, 1.0)) + assert np.allclose( + ds["stock_a"][0, :].data, np.array([0.0, 0.0, 0.0])) + assert np.allclose( + ds["stock_a"][-1, :].data, np.array([1.0, 2.0, 3.0])) + assert ds["rate_a"].shape == (3,) + + # variable attributes + assert list(model.doc.columns) == list(ds["stock_a"].attrs.keys()) + + # global attributes + assert list(ds.attrs.keys()) == [ + 'description', 'model_file', 'timestep', 'initial_time', + 'final_time'] + + model4 = pysd.read_vensim(test_variable_step) + model4.progress = False + + out_file4 = shared_tmpdir.joinpath("results4.nc") + + with catch_warnings(record=True) as w: + simplefilter("always") + model4.run(output_file=out_file4) + + with catch_warnings(record=True) as w: + simplefilter("always") + ds = xr.open_dataset(out_file4, engine="netcdf4") + + # global attributes for variable timestep + assert ds.attrs["timestep"] == "Variable" + assert ds.attrs["final_time"] == "Variable" + + # saveper final_time and time_step have time dimension + assert ds["saveper"].dims == ("time",) + assert ds["time_step"].dims == ("time",) + assert ds["final_time"].dims == ("time",) + + assert np.unique(ds["time_step"]).size == 2 + + @pytest.mark.parametrize("fmt,sep", [("csv", ","), ("tab", "\t")]) + def test_output_csv(self, fmt, sep, capsys, shared_tmpdir): + model = pysd.read_vensim(test_model_look) + model.progress = False + + out_file = shared_tmpdir.joinpath("results." + fmt) + + with catch_warnings(record=True) as w: + simplefilter("always") + model.run(output_file=out_file) + + captured = capsys.readouterr() # capture stdout + assert f"Data saved in '{out_file}'" in captured.out + + df = pd.read_csv(out_file, sep=sep) + + assert df["Time"].iloc[-1] == model["final_time"] + assert df["Time"].iloc[0] == model["initial_time"] + assert df.shape == (61, 51) + assert not df.isnull().values.any() + assert "lookup 3d time[B;Row1]" in df.columns or \ + "lookup 3d time[B,Row1]" in df.columns + + def test_dataset_handler_step_setter(self, shared_tmpdir): + model = pysd.read_vensim(test_model_look) + capture_elements = set() + results = shared_tmpdir.joinpath("results.nc") + output = ModelOutput(model, capture_elements, results) + + # Dataset handler step cannot be modified from the outside + with pytest.raises(AttributeError): + output.handler.step = 5 + + with pytest.raises(AttributeError): + output.handler.__update_step() + + assert output.handler.step == 0 + + def test_make_flat_df(self): + + df = pd.DataFrame(index=[1], columns=['elem1']) + df.at[1] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2'])] + + expected = pd.DataFrame(index=[1], data={'Elem1[B,F]': 6.}) + + return_addresses = { + 'Elem1[B,F]': ('elem1', {'Dim1': ['B'], 'Dim2': ['F']})} + + actual = DataFrameHandler.make_flat_df(df, return_addresses) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + assert_frames_close(actual, expected, rtol=1e-8, atol=1e-8) + + def test_make_flat_df_0dxarray(self): + + df = pd.DataFrame(index=[1], columns=['elem1']) + df.at[1] = [xr.DataArray(5)] + + expected = pd.DataFrame(index=[1], data={'Elem1': 5.}) + + return_addresses = {'Elem1': ('elem1', {})} + + actual = DataFrameHandler.make_flat_df( + df, return_addresses, flatten=True) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + assert_frames_close(actual, expected, rtol=1e-8, atol=1e-8) + + def test_make_flat_df_nosubs(self): + + df = pd.DataFrame(index=[1], columns=['elem1', 'elem2']) + df.at[1] = [25, 13] + + expected = pd.DataFrame(index=[1], columns=['Elem1', 'Elem2']) + expected.at[1] = [25, 13] + + return_addresses = {'Elem1': ('elem1', {}), + 'Elem2': ('elem2', {})} + + actual = DataFrameHandler.make_flat_df(df, return_addresses) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + assert all(actual['Elem1'] == expected['Elem1']) + assert all(actual['Elem2'] == expected['Elem2']) + + def test_make_flat_df_return_array(self): + """ There could be cases where we want to + return a whole section of an array - ie, by passing in only part of + the simulation dictionary. in this case, we can't force to float...""" + + df = pd.DataFrame(index=[1], columns=['elem1', 'elem2']) + df.at[1] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2']), + xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2'])] + + expected = pd.DataFrame(index=[1], columns=['Elem1[A, Dim2]', 'Elem2']) + expected.at[1] = [xr.DataArray([[1, 2, 3]], + {'Dim1': ['A'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2']), + xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2'])] + + return_addresses = { + 'Elem1[A, Dim2]': ('elem1', {'Dim1': ['A'], + 'Dim2': ['D', 'E', 'F']}), + 'Elem2': ('elem2', {})} + + actual = DataFrameHandler.make_flat_df(df, return_addresses) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + # need to assert one by one as they are xarrays + assert actual.loc[1, 'Elem1[A, Dim2]'].equals( + expected.loc[1, 'Elem1[A, Dim2]']) + assert actual.loc[1, 'Elem2'].equals(expected.loc[1, 'Elem2']) + + def test_make_flat_df_flatten(self): + + df = pd.DataFrame(index=[1], columns=['elem1', 'elem2']) + df.at[1] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2']), + xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2'])] + + expected = pd.DataFrame(index=[1], columns=[ + 'Elem1[A,D]', + 'Elem1[A,E]', + 'Elem1[A,F]', + 'Elem2[A,D]', + 'Elem2[A,E]', + 'Elem2[A,F]', + 'Elem2[B,D]', + 'Elem2[B,E]', + 'Elem2[B,F]', + 'Elem2[C,D]', + 'Elem2[C,E]', + 'Elem2[C,F]']) + + expected.at[1] = [1, 2, 3, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + return_addresses = { + 'Elem1[A,Dim2]': ('elem1', {'Dim1': ['A'], + 'Dim2': ['D', 'E', 'F']}), + 'Elem2': ('elem2', {})} + + actual = DataFrameHandler.make_flat_df( + df, return_addresses, flatten=True) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + # need to assert one by one as they are xarrays + for col in set(expected.columns): + assert actual.loc[:, col].values == expected.loc[:, col].values + + def test_make_flat_df_flatten_transposed(self): + + df = pd.DataFrame(index=[1], columns=['elem2']) + df.at[1] = [ + xr.DataArray( + [[1, 4, 7], [2, 5, 8], [3, 6, 9]], + {'Dim2': ['D', 'E', 'F'], 'Dim1': ['A', 'B', 'C']}, + ['Dim2', 'Dim1'] + ).transpose("Dim1", "Dim2") + ] + + expected = pd.DataFrame(index=[1], columns=[ + 'Elem2[A,D]', + 'Elem2[A,E]', + 'Elem2[A,F]', + 'Elem2[B,D]', + 'Elem2[B,E]', + 'Elem2[B,F]', + 'Elem2[C,D]', + 'Elem2[C,E]', + 'Elem2[C,F]']) + + expected.at[1] = [1, 2, 3, 4, 5, 6, 7, 8, 9] + + return_addresses = { + 'Elem2': ('elem2', {})} + + actual = DataFrameHandler.make_flat_df( + df, return_addresses, flatten=True) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + # need to assert one by one as they are xarrays + for col in set(expected.columns): + assert actual.loc[:, col].values == expected.loc[:, col].values + + def test_make_flat_df_times(self): + + df = pd.DataFrame(index=[1, 2], columns=['elem1']) + df['elem1'] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2']), + xr.DataArray([[2, 4, 6], [8, 10, 12], [14, 16, 19]], + {'Dim1': ['A', 'B', 'C'], + 'Dim2': ['D', 'E', 'F']}, + dims=['Dim1', 'Dim2'])] + + expected = pd.DataFrame([{'Elem1[B,F]': 6}, {'Elem1[B,F]': 12}]) + expected.index = [1, 2] + + return_addresses = {'Elem1[B,F]': ('elem1', {'Dim1': ['B'], + 'Dim2': ['F']})} + actual = DataFrameHandler.make_flat_df(df, return_addresses) + + # check all columns are in the DataFrame + assert set(actual.columns) == set(expected.columns) + assert set(actual.index) == set(expected.index) + assert all(actual['Elem1[B,F]'] == expected['Elem1[B,F]']) diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index 978f8624..b01ad6b4 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -8,8 +8,6 @@ import netCDF4 as nc from pysd.tools.benchmarking import assert_frames_close -from pysd.py_backend.output import OutputHandlerInterface, DatasetHandler, \ - DataFrameHandler, ModelOutput import pysd @@ -26,18 +24,7 @@ + "test_get_lookups_subscripted_args.mdl") test_model_data = _root.joinpath( "test-models/tests/get_data_args_3d_xls/test_get_data_args_3d_xls.mdl") -test_model_constants = _root.joinpath( - "test-models/tests/get_constants_subranges/" - "test_get_constants_subranges.mdl" -) -test_variable_step = _root.joinpath( - "test-models/tests/control_vars/" - "test_control_vars.mdl" -) -test_model_numeric_coords = _root.joinpath( - "test-models/tests/subscript_1d_arrays/" - "test_subscript_1d_arrays.mdl" -) + more_tests = _root.joinpath("more-tests/") @@ -1232,7 +1219,9 @@ def test__integrate(self, shared_tmpdir): res = out.handler.ds assert isinstance(res, nc.Dataset) assert 'teacup_temperature' in res.variables - assert np.array_equal(res["time"][:].data, np.arange(0, 5, 2)) + with catch_warnings(record=True) as w: + simplefilter("always") + assert np.array_equal(res["time"][:].data, np.arange(0, 5, 2)) res.close() def test_default_returns_with_construction_functions(self): @@ -1693,368 +1682,3 @@ def test_run_export_import_initial(self): Path('initial7.pic').unlink() assert_frames_close(stocks2, stocks) - - -class TestOutputs(): - - def test_output_handler_interface(self): - # when the class does not inherit from OutputHandlerInterface, it must - # implement all the interface to be a subclass of - # OutputHandlerInterface. - # Add any additional Handler here. - assert issubclass(DatasetHandler, OutputHandlerInterface) - assert issubclass(DataFrameHandler, OutputHandlerInterface) - - class ThatFollowsInterface: - """ - This class does not inherit from OutputHandlerInterface, but it - overrides all its methods (it follows the interface). - """ - def process_output(self, out_file): - pass - - def initialize(self, model, capture_elements): - pass - - def update(self, model, capture_elements): - pass - - def postprocess(self, **kwargs): - pass - - def add_run_elements(self, capture_elemetns): - pass - - # eventhough it does not inherit from OutputHandlerInterface, it is - # considered a subclass, because it follows the interface - assert issubclass(ThatFollowsInterface, OutputHandlerInterface) - - class IncompleteHandler: - """ - Class that does not follow the full interface - (add_run_elements is missing). - """ - def initialize(self, model, capture_elements): - pass - - def update(self, model, capture_elements): - pass - - def postprocess(self, **kwargs): - pass - - # It does not inherit from OutputHandlerInterface and does not fulfill - # its interface - assert not issubclass(IncompleteHandler, OutputHandlerInterface) - - class EmptyHandler(OutputHandlerInterface): - """ - When the class DOES inherit from OutputHandlerInterface, but does - not override all its abstract methods, then it cannot be - instantiated - """ - pass - - # it is a subclass because it inherits from it - assert issubclass(EmptyHandler, OutputHandlerInterface) - - # it cannot be instantiated because it does not override all abstract - # methods - with pytest.raises(TypeError): - empty = EmptyHandler() - - # calling methods that are not overriden returns NotImplementedError - # this should never happen, because these methods are instance methods, - # therefore the class needs to be instantiated first - with pytest.raises(NotImplementedError): - EmptyHandler.initialize(EmptyHandler, "model", "capture") - - with pytest.raises(NotImplementedError): - EmptyHandler.update(EmptyHandler, "model", "capture") - - with pytest.raises(NotImplementedError): - EmptyHandler.postprocess(EmptyHandler) - - with pytest.raises(NotImplementedError): - EmptyHandler.add_run_elements( - EmptyHandler, "model", "capture") - - def test_output_with_dimensions(self, shared_tmpdir): - model = pysd.read_vensim(test_model_look) - model.progress = False - - out_file = shared_tmpdir.joinpath("results.nc") - - with catch_warnings(record=True) as w: - simplefilter("always") - model.run(output_file=out_file) - - with nc.Dataset(out_file, "r")as ds: - assert ds.ncattrs() == [ - 'description', 'model_file', 'timestep', 'initial_time', - 'final_time'] - assert list(ds.dimensions.keys()) == ["Rows", "Dim", "time"] - # dimensions are stored as variables - assert ds["Rows"][:].size == 2 - assert "Rows" in ds.variables.keys() - assert "time" in ds.variables.keys() - # scalars do not have the time dimension - assert ds["initial_time"][:].size == 1 - # cache step variables have the "time" dimension - assert ds["lookup_1d_time"].dimensions == ("time",) - - assert ds["d2d"].dimensions == ("time", "Rows", "Dim") - assert ds["d2d"].Comment == "Missing" - assert ds["d2d"].Units == "Missing" - - # test cache run variables with dimensions - model2 = pysd.read_vensim(test_model_constants) - model2.progress = False - - out_file2 = shared_tmpdir.joinpath("results2.nc") - - with catch_warnings(record=True) as w: - simplefilter("always") - model2.run(output_file=out_file2) - - with nc.Dataset(out_file2, "r") as ds: - assert "constant" in list(ds.variables.keys()) - assert ds["constant"].dimensions == ("dim1",) - assert ds["dim1"][:].data.dtype == "S1" - - # dimension with numeric coords - model3 = pysd.read_vensim(test_model_numeric_coords) - model3.progress = False - - out_file3 = shared_tmpdir.joinpath("results3.nc") - - with catch_warnings(record=True) as w: - simplefilter("always") - model3.run(output_file=out_file3) - - with nc.Dataset(out_file3, "r")as ds: - assert np.array_equal( - ds["time"][:].data, np.arange(0.0, 101.0, 1.0)) - # coordinates get dtype=object when their length is different - assert ds["One Dimensional Subscript"][:].dtype == "O" - assert np.allclose( - ds["stock_a"][-1, :].data, np.array([1.0, 2.0, 3.0])) - assert set(model.doc.columns) == set(ds["stock_a"].ncattrs()) - - def test_variable_time_step(self, shared_tmpdir): - model = pysd.read_vensim(test_variable_step) - capture_elements = set() - results = shared_tmpdir.joinpath("results.nc") - output = ModelOutput(model, capture_elements, results) - dataset = output.handler.ds - assert dataset.timestep == "Variable" - assert dataset.final_time == "Variable" - - def test_dataset_handler_step_setter(self, shared_tmpdir): - model = pysd.read_vensim(test_model_look) - capture_elements = set() - results = shared_tmpdir.joinpath("results.nc") - output = ModelOutput(model, capture_elements, results) - - # Dataset handler step cannot be modified from the outside - with pytest.raises(AttributeError): - output.handler.step = 5 - - with pytest.raises(AttributeError): - output.handler.__update_step() - - assert output.handler.step == 0 - - def test_make_flat_df(self): - - df = pd.DataFrame(index=[1], columns=['elem1']) - df.at[1] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2'])] - - expected = pd.DataFrame(index=[1], data={'Elem1[B,F]': 6.}) - - return_addresses = { - 'Elem1[B,F]': ('elem1', {'Dim1': ['B'], 'Dim2': ['F']})} - - actual = DataFrameHandler.make_flat_df(df, return_addresses) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - assert_frames_close(actual, expected, rtol=1e-8, atol=1e-8) - - def test_make_flat_df_0dxarray(self): - - df = pd.DataFrame(index=[1], columns=['elem1']) - df.at[1] = [xr.DataArray(5)] - - expected = pd.DataFrame(index=[1], data={'Elem1': 5.}) - - return_addresses = {'Elem1': ('elem1', {})} - - actual = DataFrameHandler.make_flat_df( - df, return_addresses, flatten=True) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - assert_frames_close(actual, expected, rtol=1e-8, atol=1e-8) - - def test_make_flat_df_nosubs(self): - - df = pd.DataFrame(index=[1], columns=['elem1', 'elem2']) - df.at[1] = [25, 13] - - expected = pd.DataFrame(index=[1], columns=['Elem1', 'Elem2']) - expected.at[1] = [25, 13] - - return_addresses = {'Elem1': ('elem1', {}), - 'Elem2': ('elem2', {})} - - actual = DataFrameHandler.make_flat_df(df, return_addresses) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - assert all(actual['Elem1'] == expected['Elem1']) - assert all(actual['Elem2'] == expected['Elem2']) - - def test_make_flat_df_return_array(self): - """ There could be cases where we want to - return a whole section of an array - ie, by passing in only part of - the simulation dictionary. in this case, we can't force to float...""" - - df = pd.DataFrame(index=[1], columns=['elem1', 'elem2']) - df.at[1] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2']), - xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2'])] - - expected = pd.DataFrame(index=[1], columns=['Elem1[A, Dim2]', 'Elem2']) - expected.at[1] = [xr.DataArray([[1, 2, 3]], - {'Dim1': ['A'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2']), - xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2'])] - - return_addresses = { - 'Elem1[A, Dim2]': ('elem1', {'Dim1': ['A'], - 'Dim2': ['D', 'E', 'F']}), - 'Elem2': ('elem2', {})} - - actual = DataFrameHandler.make_flat_df(df, return_addresses) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - # need to assert one by one as they are xarrays - assert actual.loc[1, 'Elem1[A, Dim2]'].equals( - expected.loc[1, 'Elem1[A, Dim2]']) - assert actual.loc[1, 'Elem2'].equals(expected.loc[1, 'Elem2']) - - def test_make_flat_df_flatten(self): - - df = pd.DataFrame(index=[1], columns=['elem1', 'elem2']) - df.at[1] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2']), - xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2'])] - - expected = pd.DataFrame(index=[1], columns=[ - 'Elem1[A,D]', - 'Elem1[A,E]', - 'Elem1[A,F]', - 'Elem2[A,D]', - 'Elem2[A,E]', - 'Elem2[A,F]', - 'Elem2[B,D]', - 'Elem2[B,E]', - 'Elem2[B,F]', - 'Elem2[C,D]', - 'Elem2[C,E]', - 'Elem2[C,F]']) - - expected.at[1] = [1, 2, 3, 1, 2, 3, 4, 5, 6, 7, 8, 9] - - return_addresses = { - 'Elem1[A,Dim2]': ('elem1', {'Dim1': ['A'], - 'Dim2': ['D', 'E', 'F']}), - 'Elem2': ('elem2', {})} - - actual = DataFrameHandler.make_flat_df( - df, return_addresses, flatten=True) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - # need to assert one by one as they are xarrays - for col in set(expected.columns): - assert actual.loc[:, col].values == expected.loc[:, col].values - - def test_make_flat_df_flatten_transposed(self): - - df = pd.DataFrame(index=[1], columns=['elem2']) - df.at[1] = [ - xr.DataArray( - [[1, 4, 7], [2, 5, 8], [3, 6, 9]], - {'Dim2': ['D', 'E', 'F'], 'Dim1': ['A', 'B', 'C']}, - ['Dim2', 'Dim1'] - ).transpose("Dim1", "Dim2") - ] - - expected = pd.DataFrame(index=[1], columns=[ - 'Elem2[A,D]', - 'Elem2[A,E]', - 'Elem2[A,F]', - 'Elem2[B,D]', - 'Elem2[B,E]', - 'Elem2[B,F]', - 'Elem2[C,D]', - 'Elem2[C,E]', - 'Elem2[C,F]']) - - expected.at[1] = [1, 2, 3, 4, 5, 6, 7, 8, 9] - - return_addresses = { - 'Elem2': ('elem2', {})} - - actual = DataFrameHandler.make_flat_df( - df, return_addresses, flatten=True) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - # need to assert one by one as they are xarrays - for col in set(expected.columns): - assert actual.loc[:, col].values == expected.loc[:, col].values - - def test_make_flat_df_times(self): - - df = pd.DataFrame(index=[1, 2], columns=['elem1']) - df['elem1'] = [xr.DataArray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2']), - xr.DataArray([[2, 4, 6], [8, 10, 12], [14, 16, 19]], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - dims=['Dim1', 'Dim2'])] - - expected = pd.DataFrame([{'Elem1[B,F]': 6}, {'Elem1[B,F]': 12}]) - expected.index = [1, 2] - - return_addresses = {'Elem1[B,F]': ('elem1', {'Dim1': ['B'], - 'Dim2': ['F']})} - actual = DataFrameHandler.make_flat_df(df, return_addresses) - - # check all columns are in the DataFrame - assert set(actual.columns) == set(expected.columns) - assert set(actual.index) == set(expected.index) - assert all(actual['Elem1[B,F]'] == expected['Elem1[B,F]']) From 2062588fb2bc0ea8f3298a5e893f48179a5293f2 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez Date: Tue, 13 Sep 2022 19:54:25 +0200 Subject: [PATCH 37/43] Remove netcdf4 from dependencies --- .github/workflows/ci.yml | 1 - docs/getting_started.rst | 5 ++++- docs/installation.rst | 5 ++++- pysd/py_backend/output.py | 38 +++++++++++++++----------------------- requirements.txt | 2 -- tests/requirements.txt | 2 ++ 6 files changed, 25 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23f580dd..6d51229e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,6 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - pip install --upgrade pip setuptools pip install -r tests/requirements.txt pip install -e . - name: Test and coverage diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 95add588..0332dc1d 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -161,12 +161,15 @@ Storing simulation results on a file ------------------------------------ Simulation results can be stored as *.csv*, *.tab* or *.nc* (netCDF4) files by defining the desired output file path in the `output_file` argument, when calling the :py:meth:`.run` method:: - >>> model.run(output_file="results.nc") + >>> model.run(output_file="results.tab") If the `output_file` is not set, the :py:meth:`.run` method will return a :py:class:`pandas.DataFrame`. For most cases, the *.tab* file format is the safest choice. It is preferable over the *.csv* format when the model includes subscripted variables. The *.nc* format is recommended for large models, and when the user wants to keep metadata such as variable units and description. +.. warning:: + *.nc* files require :py:mod:`netcdf4` library which is an optional requirement and thus not installed automatically with the package. We recommend using :py:mod:`netcdf4` 1.6.0 or bigger, however, it will also work with :py:mod:`netcdf4` 1.5.0 or bigger. + Setting parameter values ------------------------ diff --git a/docs/installation.rst b/docs/installation.rst index 6e86595a..60b6c4a7 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -46,7 +46,6 @@ PySD builds on the core Python data analytics stack, and the following third par * Pandas * Parsimonious * xarray -* netCDF4 * xlrd * lxml * regex @@ -67,6 +66,10 @@ In order to plot model outputs as shown in :doc:`Getting started <../getting_sta * Matplotlib +In order to be able to export data to netCDF (*.nc*) files: + +* netCDF4 + These Python libraries bring additional data analytics capabilities to the analysis of SD models: diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index 8b123611..8bc7a243 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -6,7 +6,6 @@ for other output object types. """ import abc -import warnings import time as t from csv import QUOTE_NONE @@ -16,7 +15,6 @@ import numpy as np import xarray as xr import pandas as pd -import netCDF4 as nc from pysd._version import __version__ @@ -25,8 +23,8 @@ class ModelOutput(): """ - Handles different types of outputs by dispatchinging the tasks to adequate - object handlers. + Handles different types of outputs by dispatchinging the tasks + to adequate object handlers. Parameters ---------- @@ -36,6 +34,7 @@ class ModelOutput(): Which model elements to capture - uses pysafe names. out_file: str or pathlib.Path Path to the file where the results will be written. + """ valid_output_files = [".nc", ".csv", ".tab"] @@ -91,6 +90,7 @@ def handle(self, out_file): Returns ------- handler + """ handler = self.process_output(out_file) @@ -161,6 +161,7 @@ def __init__(self, next): self.out_file = None self.ds = None self._step = 0 + self.nc = __import__("netCDF4") @property def step(self): @@ -213,7 +214,7 @@ def initialize(self, model, capture_elements): None """ - self.ds = nc.Dataset(self.out_file, "w") + self.ds = self.nc.Dataset(self.out_file, "w") # defining global attributes self.ds.description = "Results for simulation run on " \ @@ -230,20 +231,16 @@ def initialize(self, model, capture_elements): coords = np.array(coords) # create dimension self.ds.createDimension(dim_name, len(coords)) - # length of the longest string in the coords max_str_len = len(max(coords, key=len)) - # create variable for the dimension var = self.ds.createVariable( dim_name, f"S{max_str_len}", (dim_name,)) - # assigning coords to dimension var[:] = coords # creating the time dimension as unlimited self.ds.createDimension("time", None) - # creating variables self.__create_ds_vars(model, capture_elements) @@ -262,11 +259,10 @@ def update(self, model, capture_elements): Returns ------- None + """ for key in capture_elements: - comp = model[key] - if isinstance(comp, xr.DataArray): self.ds[key][self.step, :] = comp.values else: @@ -290,11 +286,10 @@ def __update_run_elements(self, model, capture_elements): Returns ------- None + """ for key in capture_elements: - comp = model[key] - if isinstance(comp, xr.DataArray): self.ds[key][:] = comp.values else: @@ -309,7 +304,6 @@ def postprocess(self, **kwargs): None """ self.ds.close() - print(f"Results stored in {self.out_file}") def add_run_elements(self, model, capture_elements): @@ -326,10 +320,10 @@ def add_run_elements(self, model, capture_elements): Returns ------- None + """ # creating variables in capture_elements self.__create_ds_vars(model, capture_elements, time_dim=False) - self.__update_run_elements(model, capture_elements) def __create_ds_vars(self, model, capture_elements, time_dim=True): @@ -353,22 +347,19 @@ def __create_ds_vars(self, model, capture_elements, time_dim=True): """ kwargs = dict() - if tuple(nc.__version__.split(".")) >= ('1', '6', '0'): - kwargs.update({"compression": "zlib"}) + if tuple(self.nc.__version__.split(".")) >= ('1', '6', '0'): + kwargs["compression"] = "zlib" for key in capture_elements: comp = model[key] - dims = () - + dims = tuple() if isinstance(comp, xr.DataArray): dims = tuple(comp.dims) - if time_dim: dims = ("time",) + dims var = self.ds.createVariable(key, "f8", dims, **kwargs) - # adding metadata for each var from the model.doc for col in model.doc.columns: var.setncattr( @@ -444,8 +435,8 @@ def update(self, model, capture_elements): Returns ------- None - """ + """ self.ds.at[model.time.round()] = [ getattr(model.components, key)() for key in capture_elements] @@ -459,6 +450,7 @@ def postprocess(self, **kwargs): ------- ds: pandas.DataFrame Simulation results stored as a pandas DataFrame. + """ # delete time column as it was created only for avoiding errors # of appending data. See previous TODO. @@ -490,7 +482,6 @@ def __save_to_file(self, output): None """ - if self.out_file.suffix == ".tab": sep = "\t" else: @@ -518,6 +509,7 @@ def add_run_elements(self, model, capture_elements): Returns ------- None + """ nx = len(self.ds.index) for element in capture_elements: diff --git a/requirements.txt b/requirements.txt index cf9a1aca..e0947fec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,6 @@ pandas parsimonious xarray -netCDF4==1.5; platform_system == 'Windows' and python_version == "3.7" -netCDF4==1.6; platform_system != 'Windows' or python_version != "3.7" xlrd lxml regex diff --git a/tests/requirements.txt b/tests/requirements.txt index 18dc6a94..833d24f2 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -4,3 +4,5 @@ pytest-mock coverage coveralls psutil +netCDF4==1.5; platform_system == 'Windows' and python_version == "3.7" +netCDF4==1.6; platform_system != 'Windows' or python_version != "3.7" From 53b8b9715ce9701785ecafacbc1e939ca51d09d3 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez <61285767+enekomartinmartinez@users.noreply.github.com> Date: Tue, 13 Sep 2022 23:01:28 +0200 Subject: [PATCH 38/43] Update getting_started.rst --- docs/getting_started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 0332dc1d..ad00c754 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -168,7 +168,7 @@ If the `output_file` is not set, the :py:meth:`.run` method will return a :py:cl For most cases, the *.tab* file format is the safest choice. It is preferable over the *.csv* format when the model includes subscripted variables. The *.nc* format is recommended for large models, and when the user wants to keep metadata such as variable units and description. .. warning:: - *.nc* files require :py:mod:`netcdf4` library which is an optional requirement and thus not installed automatically with the package. We recommend using :py:mod:`netcdf4` 1.6.0 or bigger, however, it will also work with :py:mod:`netcdf4` 1.5.0 or bigger. + *.nc* files require :py:mod:`netcdf4` library which is an optional requirement and thus not installed automatically with the package. We recommend using :py:mod:`netcdf4` 1.6.0 or above, however, it will also work with :py:mod:`netcdf4` 1.5.0 or above. Setting parameter values From 91806bdf16a82c0f07d2c625434740fff185ef11 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez Date: Thu, 15 Sep 2022 18:26:09 +0200 Subject: [PATCH 39/43] Solve bug of variables defined with a partial range of subscripts --- docs/whats_new.rst | 2 ++ .../python/python_expressions_builder.py | 6 ++--- pysd/builders/python/python_model_builder.py | 5 ++++ .../pytest_integration_vensim_pathway.py | 4 +++ tests/pytest_pysd/pytest_output.py | 27 ++++++++++++++++++- tests/test-models | 2 +- 6 files changed, 41 insertions(+), 5 deletions(-) diff --git a/docs/whats_new.rst b/docs/whats_new.rst index 1ea3764a..151ad0c0 100644 --- a/docs/whats_new.rst +++ b/docs/whats_new.rst @@ -17,6 +17,8 @@ Deprecations Bug fixes ~~~~~~~~~ - Fix bug when a WITH LOOKUPS argument has subscripts. (`@enekomartinmartinez `_) +- Fix bug of exportig csv files with multiple subscripts variables. (`@rogersamso `_) +- Fix bug of missing dimensions in variables defined with not all the subscripts of a range (:issue:`364`). (`@enekomartinmartinez `_) Documentation ~~~~~~~~~~~~~ diff --git a/pysd/builders/python/python_expressions_builder.py b/pysd/builders/python/python_expressions_builder.py index 1c0b76fc..e4df5a51 100644 --- a/pysd/builders/python/python_expressions_builder.py +++ b/pysd/builders/python/python_expressions_builder.py @@ -1712,15 +1712,15 @@ def build(self, arguments: dict) -> Union[BuildAST, None]: arguments["name"] = self.section.namespace.make_python_identifier( self.element.identifier, prefix="_hardcodedlookup") - - arguments["final_subs"] = self.element.subs_dict + arguments["final_subs"] = "%(final_subs)s" self.element.objects["hardcoded_lookups"] = { "name": arguments["name"], "expression": "%(name)s = HardcodedLookups(%(x)s, %(y)s, " "%(subscripts)s, '%(interp)s', " "%(final_subs)s, '%(name)s')" - % arguments + % arguments, + "final_subs": self.element.subs_dict } return BuildAST( diff --git a/pysd/builders/python/python_model_builder.py b/pysd/builders/python/python_model_builder.py index 4a20e21a..637571f6 100644 --- a/pysd/builders/python/python_model_builder.py +++ b/pysd/builders/python/python_model_builder.py @@ -639,6 +639,11 @@ def build_element(self) -> None: else: self.pre_expression = "" # NUMPY: reshape to the final shape if needed + # include full subscript range for objects defined with a + # partial range (issue #363) + for value in self.objects.values(): + if value["expression"] is not None and "final_subs" in value: + value["final_subs"] = self.subs_dict # expressions[0]["expr"].reshape(self.section.subscripts, {}) if not expressions[0]["expr"].subscripts and self.subscripts: # Updimension the return value to an array diff --git a/tests/pytest_integration/pytest_integration_vensim_pathway.py b/tests/pytest_integration/pytest_integration_vensim_pathway.py index 2663c51d..71c3fb29 100644 --- a/tests/pytest_integration/pytest_integration_vensim_pathway.py +++ b/tests/pytest_integration/pytest_integration_vensim_pathway.py @@ -314,6 +314,10 @@ "folder": "parentheses", "file": "test_parens.mdl" }, + "partial_range_definitions": { + "folder": "partial_range_definitions", + "file": "test_partial_range_definitions.mdl" + }, "reference_capitalization": { "folder": "reference_capitalization", "file": "test_reference_capitalization.mdl" diff --git a/tests/pytest_pysd/pytest_output.py b/tests/pytest_pysd/pytest_output.py index b251f8be..a0d73874 100644 --- a/tests/pytest_pysd/pytest_output.py +++ b/tests/pytest_pysd/pytest_output.py @@ -31,6 +31,10 @@ "test-models/tests/control_vars/" "test_control_vars.mdl" ) +test_partial_definitions = _root.joinpath( + "test-models/tests/partial_range_definitions/" + "test_partial_range_definitions.mdl" +) class TestOutput(): @@ -99,7 +103,7 @@ class EmptyHandler(OutputHandlerInterface): # it cannot be instantiated because it does not override all abstract # methods with pytest.raises(TypeError): - empty = EmptyHandler() + EmptyHandler() # calling methods that are not overriden returns NotImplementedError # this should never happen, because these methods are instance methods, @@ -234,6 +238,27 @@ def test_output_nc(self, shared_tmpdir): assert np.unique(ds["time_step"]).size == 2 + def test_output_nc2(self, shared_tmpdir): + # dimension with numeric coords + with catch_warnings(record=True) as w: + simplefilter("always") + model5 = pysd.read_vensim(test_partial_definitions) + model5.progress = False + + out_file5 = shared_tmpdir.joinpath("results5.nc") + + with catch_warnings(record=True) as w: + simplefilter("always") + model5.run(output_file=out_file5) + + # using xarray instead of netCDF4 to load the dataset + + with catch_warnings(record=True) as w: + simplefilter("always") + ds = xr.open_dataset(out_file5, engine="netcdf4") + + print(ds) + @pytest.mark.parametrize("fmt,sep", [("csv", ","), ("tab", "\t")]) def test_output_csv(self, fmt, sep, capsys, shared_tmpdir): model = pysd.read_vensim(test_model_look) diff --git a/tests/test-models b/tests/test-models index be07310e..025d33b3 160000 --- a/tests/test-models +++ b/tests/test-models @@ -1 +1 @@ -Subproject commit be07310e8d45def1c7d2d973bac630274fd158b7 +Subproject commit 025d33b360ade3690bd59824837d46a80ad10390 From c9142dd4235b0fca0020aecb2aba01ce2965ed50 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez Date: Thu, 15 Sep 2022 23:36:19 +0200 Subject: [PATCH 40/43] Break test using parametrize --- pysd/py_backend/output.py | 3 + tests/pytest_pysd/pytest_output.py | 329 ++++++++++++++++------------- tests/test-models | 2 +- 3 files changed, 182 insertions(+), 152 deletions(-) diff --git a/pysd/py_backend/output.py b/pysd/py_backend/output.py index 8bc7a243..8997ec41 100644 --- a/pysd/py_backend/output.py +++ b/pysd/py_backend/output.py @@ -362,6 +362,9 @@ def __create_ds_vars(self, model, capture_elements, time_dim=True): var = self.ds.createVariable(key, "f8", dims, **kwargs) # adding metadata for each var from the model.doc for col in model.doc.columns: + if col in ["Subscripts", "Limits"]: + # pass those that cannot be saved as attributes + continue var.setncattr( col, model.doc.loc[model.doc["Py Name"] == key, col].values[0] diff --git a/tests/pytest_pysd/pytest_output.py b/tests/pytest_pysd/pytest_output.py index a0d73874..c4667b29 100644 --- a/tests/pytest_pysd/pytest_output.py +++ b/tests/pytest_pysd/pytest_output.py @@ -1,5 +1,5 @@ from pathlib import Path -from warnings import simplefilter, catch_warnings +import shutil import pytest import numpy as np @@ -14,29 +14,40 @@ import pysd -_root = Path(__file__).parent.parent - -test_model_look = _root.joinpath( +test_model_look = Path( "test-models/tests/get_lookups_subscripted_args/" - + "test_get_lookups_subscripted_args.mdl") -test_model_constants = _root.joinpath( + "test_get_lookups_subscripted_args.mdl" +) +test_model_constants = Path( "test-models/tests/get_constants_subranges/" "test_get_constants_subranges.mdl" ) -test_model_numeric_coords = _root.joinpath( +test_model_numeric_coords = Path( "test-models/tests/subscript_1d_arrays/" "test_subscript_1d_arrays.mdl" ) -test_variable_step = _root.joinpath( +test_variable_step = Path( "test-models/tests/control_vars/" "test_control_vars.mdl" ) -test_partial_definitions = _root.joinpath( +test_partial_definitions = Path( "test-models/tests/partial_range_definitions/" "test_partial_range_definitions.mdl" ) +@pytest.fixture +@pytest.mark.filterwarnings("ignore") +def model(_root, tmp_path, model_path): + """ + Copy model to the tmp_path and translate it + """ + + target = tmp_path / model_path.parent + shutil.copytree(_root / model_path.parent, target) + return pysd.read_vensim(target / model_path.name) + + class TestOutput(): def test_output_handler_interface(self): @@ -124,151 +135,167 @@ class EmptyHandler(OutputHandlerInterface): EmptyHandler.add_run_elements( EmptyHandler, "model", "capture") - def test_output_nc(self, shared_tmpdir): - model = pysd.read_vensim(test_model_look) - model.progress = False - - out_file = shared_tmpdir.joinpath("results.nc") - - with catch_warnings(record=True) as w: - simplefilter("always") - model.run(output_file=out_file) + @pytest.mark.parametrize( + "model_path,dims,values", + [ + ( + test_model_look, + { + "Rows": 2, + "Dim": 2, + "time": 61 + }, + { + "lookup_1d_time": (("time",), None), + "d2d": (("time", "Rows", "Dim"), None), + "initial_time": (tuple(), 0), + "final_time": (tuple(), 30), + "saveper": (tuple(), 0.5), + "time_step": (tuple(), 0.5) + } + + ), + ( + test_model_constants, + { + "dim1": 5, + "dim1a": 2, + "dim1c": 3, + 'time': 2 + }, + { + "constant": ( + ("dim1",), + np.array([0., 0., 1., 15., 50.]) + ) + } + ), + ( + test_model_numeric_coords, + { + "One Dimensional Subscript": 3, + 'time': 101 + }, + { + "rate_a": ( + ("One Dimensional Subscript",), + np.array([0.01, 0.02, 0.03])), + "stock_a": ( + ("time", "One Dimensional Subscript"), + np.array([ + np.arange(0, 1.0001, 0.01), + np.arange(0, 2.0001, 0.02), + np.arange(0, 3.0001, 0.03)], + dtype=float).transpose()), + "time": (("time",), np.arange(0.0, 101.0, 1.0)) + } + ), + ( + test_variable_step, + { + "time": 25 + }, + { + "final_time": ( + ("time",), + np.array([ + 10., 10., 10., 10., 10., 10., + 50., 50., 50., 50., 50., 50., + 50., 50., 50., 50., 50., 50., + 50., 50., 50., 50., 50., 50., 50. + ])), + "initial_time": ( + ("time",), + np.array([ + 0., 0., 0., 0.2, 0.2, 0.2, + 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, + 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, + 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2 + ])), + "time_step": ( + ("time",), + np.array([ + 1., 1., 1., 1., 0.5, 0.5, + 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, + 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, + 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 + ])), + "saveper": ( + ("time",), + np.array([ + 1., 1., 1., 1., 0.5, 0.5, + 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, + 0.5, 0.5, 0.5, 0.5, 5., 5., + 5., 5., 5., 5., 5., 5., 5.])) + } + ), + ( + test_partial_definitions, + { + "my range": 5, + "time": 11 + }, + { + "partial_data": (("time", "my range"), None), + "partial_constants": (("my range",), None) + } + ) + ], + ids=["lookups", "constants", "numeric_coords", "variable_step", + "partial_definitions"] + ) + @pytest.mark.filterwarnings("ignore") + def test_output_nc(self, tmp_path, model, dims, values): + + out_file = tmp_path.joinpath("results.nc") + + model.run(output_file=out_file) with nc.Dataset(out_file, "r") as ds: assert ds.ncattrs() == [ 'description', 'model_file', 'timestep', 'initial_time', 'final_time'] - assert list(ds.dimensions.keys()) == ["Rows", "Dim", "time"] + assert list(ds.dimensions) == list(dims) # dimensions are stored as variables - assert ds["Rows"].size == 2 - assert "Rows" in ds.variables.keys() - assert "time" in ds.variables.keys() - # scalars do not have the time dimension - assert ds["initial_time"].size == 1 - # cache step variables have the "time" dimension - assert ds["lookup_1d_time"].dimensions == ("time",) - - assert ds["d2d"].dimensions == ("time", "Rows", "Dim") - - with catch_warnings(record=True) as w: - simplefilter("always") - assert ds["d2d"].Comment == "Missing" - assert ds["d2d"].Units == "Missing" - - # test cache run variables with dimensions - model2 = pysd.read_vensim(test_model_constants) - model2.progress = False - - out_file2 = shared_tmpdir.joinpath("results2.nc") - - with catch_warnings(record=True) as w: - simplefilter("always") - model2.run(output_file=out_file2) - - with nc.Dataset(out_file2, "r") as ds: - assert ds["time_step"].size == 1 - assert "constant" in list(ds.variables.keys()) - assert ds["constant"].dimensions == ("dim1",) - - with catch_warnings(record=True) as w: - simplefilter("always") - assert ds["dim1"][:].data.dtype == "S1" - - # dimension with numeric coords - model3 = pysd.read_vensim(test_model_numeric_coords) - model3.progress = False - - out_file3 = shared_tmpdir.joinpath("results3.nc") - - with catch_warnings(record=True) as w: - simplefilter("always") - model3.run(output_file=out_file3) - - # using xarray instead of netCDF4 to load the dataset - - with catch_warnings(record=True) as w: - simplefilter("always") - ds = xr.open_dataset(out_file3, engine="netcdf4") - - assert "time" in ds.dims - assert ds["rate_a"].dims == ("One Dimensional Subscript",) - assert ds["stock_a"].dims == ("time", "One Dimensional Subscript") - - # coordinates get dtype=object when their length is different - assert ds["One Dimensional Subscript"].data.dtype == "O" - - # check data - assert np.array_equal( - ds["time"].data, np.arange(0.0, 101.0, 1.0)) - assert np.allclose( - ds["stock_a"][0, :].data, np.array([0.0, 0.0, 0.0])) - assert np.allclose( - ds["stock_a"][-1, :].data, np.array([1.0, 2.0, 3.0])) - assert ds["rate_a"].shape == (3,) - - # variable attributes - assert list(model.doc.columns) == list(ds["stock_a"].attrs.keys()) - - # global attributes - assert list(ds.attrs.keys()) == [ - 'description', 'model_file', 'timestep', 'initial_time', - 'final_time'] - - model4 = pysd.read_vensim(test_variable_step) - model4.progress = False - - out_file4 = shared_tmpdir.joinpath("results4.nc") - - with catch_warnings(record=True) as w: - simplefilter("always") - model4.run(output_file=out_file4) - - with catch_warnings(record=True) as w: - simplefilter("always") - ds = xr.open_dataset(out_file4, engine="netcdf4") - - # global attributes for variable timestep - assert ds.attrs["timestep"] == "Variable" - assert ds.attrs["final_time"] == "Variable" - - # saveper final_time and time_step have time dimension - assert ds["saveper"].dims == ("time",) - assert ds["time_step"].dims == ("time",) - assert ds["final_time"].dims == ("time",) - - assert np.unique(ds["time_step"]).size == 2 - - def test_output_nc2(self, shared_tmpdir): - # dimension with numeric coords - with catch_warnings(record=True) as w: - simplefilter("always") - model5 = pysd.read_vensim(test_partial_definitions) - model5.progress = False - - out_file5 = shared_tmpdir.joinpath("results5.nc") - - with catch_warnings(record=True) as w: - simplefilter("always") - model5.run(output_file=out_file5) - - # using xarray instead of netCDF4 to load the dataset - - with catch_warnings(record=True) as w: - simplefilter("always") - ds = xr.open_dataset(out_file5, engine="netcdf4") - - print(ds) - - @pytest.mark.parametrize("fmt,sep", [("csv", ","), ("tab", "\t")]) - def test_output_csv(self, fmt, sep, capsys, shared_tmpdir): - model = pysd.read_vensim(test_model_look) - model.progress = False - - out_file = shared_tmpdir.joinpath("results." + fmt) - - with catch_warnings(record=True) as w: - simplefilter("always") - model.run(output_file=out_file) + for dim, n in dims.items(): + # check dimension size + assert ds[dim].size == n + assert dim in ds.variables.keys() + # check dimension type + if dim != "time": + assert ds[dim].dtype in ["S1", str] + else: + assert ds[dim].dtype == float + + for var, (dim, val) in values.items(): + # check variable dimensions + assert ds[var].dimensions == dim + if val is not None: + # check variable values if given + assert np.all(np.isclose(ds[var][:].data, val)) + + # Check variable attributes + doc = model.doc + doc.set_index("Py Name", drop=False, inplace=True) + doc.drop(columns=["Subscripts", "Limits"], inplace=True) + + for var in doc["Py Name"]: + if doc.loc[var, "Type"] == "Lookup": + continue + for key in doc.columns: + assert getattr(ds[var], key) == (doc.loc[var, key] + or "Missing") + + @pytest.mark.parametrize( + "model_path,fmt,sep", + [ + (test_model_look, "csv", ","), + (test_model_look, "tab", "\t")]) + @pytest.mark.filterwarnings("ignore") + def test_output_csv(self, fmt, sep, capsys, model, tmp_path): + out_file = tmp_path.joinpath("results." + fmt) + + model.run(output_file=out_file) captured = capsys.readouterr() # capture stdout assert f"Data saved in '{out_file}'" in captured.out @@ -282,10 +309,10 @@ def test_output_csv(self, fmt, sep, capsys, shared_tmpdir): assert "lookup 3d time[B;Row1]" in df.columns or \ "lookup 3d time[B,Row1]" in df.columns - def test_dataset_handler_step_setter(self, shared_tmpdir): - model = pysd.read_vensim(test_model_look) + @pytest.mark.parametrize("model_path", [test_model_look]) + def test_dataset_handler_step_setter(self, tmp_path, model): capture_elements = set() - results = shared_tmpdir.joinpath("results.nc") + results = tmp_path.joinpath("results.nc") output = ModelOutput(model, capture_elements, results) # Dataset handler step cannot be modified from the outside diff --git a/tests/test-models b/tests/test-models index 025d33b3..03e2369c 160000 --- a/tests/test-models +++ b/tests/test-models @@ -1 +1 @@ -Subproject commit 025d33b360ade3690bd59824837d46a80ad10390 +Subproject commit 03e2369cc68c51cf8acb6821601cc1c26c79d47d From 2ded59907f5539c47c04ba62d8d10b3acd789f22 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez Date: Fri, 16 Sep 2022 15:41:53 +0200 Subject: [PATCH 41/43] Bug fixes and improve tests --- docs/whats_new.rst | 5 +- pysd/py_backend/model.py | 34 +- tests/conftest.py | 30 +- tests/pytest_pysd/pytest_output.py | 25 +- tests/pytest_pysd/pytest_pysd.py | 1129 +++++++++++++--------------- 5 files changed, 574 insertions(+), 649 deletions(-) diff --git a/docs/whats_new.rst b/docs/whats_new.rst index 151ad0c0..dace4401 100644 --- a/docs/whats_new.rst +++ b/docs/whats_new.rst @@ -19,6 +19,7 @@ Bug fixes - Fix bug when a WITH LOOKUPS argument has subscripts. (`@enekomartinmartinez `_) - Fix bug of exportig csv files with multiple subscripts variables. (`@rogersamso `_) - Fix bug of missing dimensions in variables defined with not all the subscripts of a range (:issue:`364`). (`@enekomartinmartinez `_) +- Fix bug when running a model with variable final time or time step and progressbar (:issue:`361`). (`@enekomartinmartinez `_) Documentation ~~~~~~~~~~~~~ @@ -31,7 +32,9 @@ Performance Internal Changes ~~~~~~~~~~~~~~~~ - Make PySD work with :py:mod:`parsimonius` 0.10.0. (`@enekomartinmartinez `_) -- Add netCDF4 and hdf5 dependencies. (`@rogersamso `_) +- Add netCDF4 dependency for tests. (`@rogersamso `_) +- Improve warning message when replacing a stock with a parameter. (`@enekomartinmartinez `_) +- Include more pytest parametrizations in some test and make them translate the models in temporary directories. (`@enekomartinmartinez `_) v3.6.1 (2022/09/05) diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py index c4f10b3f..1a13e9f4 100644 --- a/pysd/py_backend/model.py +++ b/pysd/py_backend/model.py @@ -485,7 +485,7 @@ def export(self, file_name): Parameters ---------- - file_name: str + file_name: str or pathlib.Path Name of the file to export the values. """ @@ -512,7 +512,7 @@ def import_pickle(self, file_name): Parameters ---------- - file_name: str + file_name: str or pathlib.Path Name of the file to import the values from. """ @@ -781,8 +781,8 @@ def set_components(self, params, new=False): # this won't handle other statefuls... if '_integ_' + func_name in dir(self.components): - warnings.warn("Replacing the equation of stock" - + "{} with params".format(key), + warnings.warn("Replacing the equation of stock " + "'{}' with params...".format(key), stacklevel=2) new_function.__name__ = func_name @@ -1134,8 +1134,6 @@ def run(self, params=None, return_columns=None, return_timestamps=None, if reload: self.reload() - self.progress = progress - self.time.add_return_timestamps(return_timestamps) if self.time.return_timestamps is not None and not final_time: # if not final time given the model will end in the list @@ -1156,6 +1154,18 @@ def run(self, params=None, return_columns=None, return_timestamps=None, self.set_initial_condition(initial_condition) + # set progressbar + if progress and (self.cache_type["final_time"] == "step" or + self.cache_type["time_step"] == "step"): + warnings.warn( + "The progressbar is not compatible with dynamic " + "final time or time step. Both variables must be " + "constants to prompt progress." + ) + progress = False + + self.progress = progress + if return_columns is None or isinstance(return_columns, str): return_columns = self._default_return_columns(return_columns) @@ -1625,7 +1635,7 @@ def set_initial_condition(self, initial_condition): Parameters ---------- - initial_condition : str or (float, dict) + initial_condition : str or (float, dict) or pathlib.Path The starting time, and the state of the system (the values of all the stocks) at that starting time. 'original' or 'o'uses model-file specified initial condition. 'current' or 'c' uses @@ -1647,19 +1657,21 @@ def set_initial_condition(self, initial_condition): model.set_initial_value() """ + if isinstance(initial_condition, str)\ + and initial_condition.lower() not in ["original", "o", + "current", "c"]: + initial_condition = Path(initial_condition) if isinstance(initial_condition, tuple): self.initialize() self.set_initial_value(*initial_condition) + elif isinstance(initial_condition, Path): + self.import_pickle(initial_condition) elif isinstance(initial_condition, str): if initial_condition.lower() in ["original", "o"]: self.time.set_control_vars( initial_time=self.components._control_vars["initial_time"]) self.initialize() - elif initial_condition.lower() in ["current", "c"]: - pass - else: - self.import_pickle(initial_condition) else: raise TypeError( "Invalid initial conditions. " diff --git a/tests/conftest.py b/tests/conftest.py index b4fd902b..0019e9ad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,13 @@ -import pytest +import shutil from pathlib import Path +import pytest +from pysd import read_vensim, read_xmile, load +from pysd.translators.vensim.vensim_utils import supported_extensions as\ + vensim_extensions +from pysd.translators.xmile.xmile_utils import supported_extensions as\ + xmile_extensions + @pytest.fixture(scope="session") def _root(): @@ -18,3 +25,24 @@ def _test_models(_root): def shared_tmpdir(tmpdir_factory): # shared temporary directory for each class return Path(tmpdir_factory.mktemp("shared")) + + +@pytest.fixture +def model(_root, tmp_path, model_path): + """ + Copy model to the tmp_path and translate it + """ + assert (_root / model_path).exists(), "The model doesn't exist" + + target = tmp_path / model_path.parent.name + new_path = target / model_path.name + shutil.copytree(_root / model_path.parent, target) + + if model_path.suffix.lower() in vensim_extensions: + return read_vensim(new_path) + elif model_path.suffix.lower() in xmile_extensions: + return read_xmile(new_path) + elif model_path.suffix.lower() == ".py": + return load(new_path) + else: + return ValueError("Invalid model") diff --git a/tests/pytest_pysd/pytest_output.py b/tests/pytest_pysd/pytest_output.py index c4667b29..de45856f 100644 --- a/tests/pytest_pysd/pytest_output.py +++ b/tests/pytest_pysd/pytest_output.py @@ -1,5 +1,4 @@ from pathlib import Path -import shutil import pytest import numpy as np @@ -11,8 +10,6 @@ from pysd.py_backend.output import OutputHandlerInterface, DatasetHandler, \ DataFrameHandler, ModelOutput -import pysd - test_model_look = Path( "test-models/tests/get_lookups_subscripted_args/" @@ -36,18 +33,6 @@ ) -@pytest.fixture -@pytest.mark.filterwarnings("ignore") -def model(_root, tmp_path, model_path): - """ - Copy model to the tmp_path and translate it - """ - - target = tmp_path / model_path.parent - shutil.copytree(_root / model_path.parent, target) - return pysd.read_vensim(target / model_path.name) - - class TestOutput(): def test_output_handler_interface(self): @@ -135,6 +120,16 @@ class EmptyHandler(OutputHandlerInterface): EmptyHandler.add_run_elements( EmptyHandler, "model", "capture") + @pytest.mark.parametrize("model_path", [test_model_look]) + def test_invalid_output_file(self, model): + error_message = "Paths must be strings or pathlib Path objects." + with pytest.raises(TypeError, match=error_message): + model.run(output_file=1234) + + error_message = "Unsupported output file format .txt" + with pytest.raises(ValueError, match=error_message): + model.run(output_file="file.txt") + @pytest.mark.parametrize( "model_path,dims,values", [ diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index b01ad6b4..a71e227b 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -1,5 +1,4 @@ from pathlib import Path -from warnings import simplefilter, catch_warnings import pytest import pandas as pd @@ -14,19 +13,17 @@ # TODO replace test paths by fixtures and translate and run the models # in temporal directories -_root = Path(__file__).parent.parent - -test_model = _root.joinpath("test-models/samples/teacup/teacup.mdl") -test_model_subs = _root.joinpath( +test_model = Path("test-models/samples/teacup/teacup.mdl") +test_model_subs = Path( "test-models/tests/subscript_2d_arrays/test_subscript_2d_arrays.mdl") -test_model_look = _root.joinpath( +test_model_look = Path( "test-models/tests/get_lookups_subscripted_args/" + "test_get_lookups_subscripted_args.mdl") -test_model_data = _root.joinpath( +test_model_data = Path( "test-models/tests/get_data_args_3d_xls/test_get_data_args_3d_xls.mdl") -more_tests = _root.joinpath("more-tests/") +more_tests = Path("more-tests") test_model_constant_pipe = more_tests.joinpath( "constant_pipeline/test_constant_pipeline.mdl") @@ -34,8 +31,8 @@ class TestPySD(): - def test_run(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_run(self, model): stocks = model.run() # return a dataframe assert isinstance(stocks, pd.DataFrame) @@ -46,7 +43,7 @@ def test_run(self): # there are no null values in the set assert stocks.notnull().all().all() - def test_run_ignore_missing(self): + def test_run_ignore_missing(self, _root): model_mdl = _root.joinpath( 'test-models/tests/get_with_missing_values_xlsx/' + 'test_get_with_missing_values_xlsx.mdl') @@ -73,32 +70,46 @@ def test_run_ignore_missing(self): # errors for missing values pysd.load(model_py, missing_values="raise") - def test_run_includes_last_value(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_run_includes_last_value(self, model): res = model.run() assert res.index[-1] == model.components.final_time() - def test_run_build_timeseries(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_run_build_timeseries(self, model): res = model.run(final_time=7, time_step=2, initial_condition=(3, {})) actual = list(res.index) expected = [3.0, 5.0, 7.0] assert actual == expected - def test_run_progress(self): + @pytest.mark.parametrize("model_path", [test_model]) + def test_run_progress(self, model): # same as test_run but with progressbar - model = pysd.read_vensim(test_model) stocks = model.run(progress=True) assert isinstance(stocks, pd.DataFrame) assert "Teacup Temperature" in stocks.columns.values assert len(stocks) > 3 assert stocks.notnull().all().all() - def test_run_return_timestamps(self): - """Addresses https://github.com/JamesPHoughton/pysd/issues/17""" + @pytest.mark.parametrize( + "model_path", + [Path("test-models/tests/control_vars/test_control_vars.mdl")]) + def test_run_progress_dynamic(self, model): + # same as test_run but with progressbar + warn_message = r"The progressbar is not compatible with dynamic "\ + r"final time or time step\. Both variables must be "\ + r"constants to prompt progress\." + with pytest.warns(UserWarning, match=warn_message): + stocks = model.run(progress=True) + assert isinstance(stocks, pd.DataFrame) + for var in ["FINAL TIME", "TIME STEP"]: + # assert that control variables have change + assert len(np.unique(stocks[var].values)) > 1 - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_run_return_timestamps(self, model): + """Addresses https://github.com/JamesPHoughton/pysd/issues/17""" timestamps = np.random.randint(1, 5, 5).cumsum() stocks = model.run(return_timestamps=timestamps) assert (stocks.index.values == timestamps).all() @@ -147,51 +158,50 @@ def test_run_return_timestamps(self): assert 0.95 not in stocks.index assert 0.55 not in stocks.index - def test_run_return_timestamps_past_final_time(self): - """ If the user enters a timestamp that is longer than the euler + @pytest.mark.parametrize("model_path", [test_model]) + def test_run_return_timestamps_past_final_time(self, model): + """ + If the user enters a timestamp that is longer than the euler timeseries that is defined by the normal model file, should - extend the euler series to the largest timestamp""" - - model = pysd.read_vensim(test_model) + extend the euler series to the largest timestamp + """ return_timestamps = list(range(0, 100, 10)) stocks = model.run(return_timestamps=return_timestamps) assert return_timestamps == list(stocks.index) - def test_return_timestamps_with_range(self): + @pytest.mark.parametrize("model_path", [test_model]) + def test_return_timestamps_with_range(self, model): """ Tests that return timestamps may receive a 'range'. It will be cast to a numpy array in the end... """ - - model = pysd.read_vensim(test_model) return_timestamps = range(0, 31, 10) stocks = model.run(return_timestamps=return_timestamps) assert list(return_timestamps) == list(stocks.index) - def test_run_return_columns_original_names(self): - """Addresses https://github.com/JamesPHoughton/pysd/issues/26 - - Also checks that columns are returned in the correct order""" - - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_run_return_columns_original_names(self, model): + """ + Addresses https://github.com/JamesPHoughton/pysd/issues/26 + - Also checks that columns are returned in the correct order + """ return_columns = ["Room Temperature", "Teacup Temperature"] result = model.run(return_columns=return_columns) assert set(result.columns) == set(return_columns) - def test_run_return_columns_step(self): + @pytest.mark.parametrize("model_path", [test_model]) + def test_run_return_columns_step(self, model): """ Return only cache 'step' variables """ - model = pysd.read_vensim(test_model) result = model.run(return_columns='step') assert set(result.columns)\ == {'Teacup Temperature', 'Heat Loss to Room'} - def test_run_reload(self): - """ Addresses https://github.com/JamesPHoughton/pysd/issues/99""" - - model = pysd.read_vensim(test_model) - + @pytest.mark.parametrize("model_path", [test_model]) + def test_run_reload(self, model): + """Addresses https://github.com/JamesPHoughton/pysd/issues/99""" result0 = model.run() result1 = model.run(params={"Room Temperature": 1000}) result2 = model.run() @@ -201,38 +211,23 @@ def test_run_reload(self): assert not (result0 == result1).all().all() assert (result1 == result2).all().all() - def test_run_return_columns_pysafe_names(self): + @pytest.mark.parametrize("model_path", [test_model]) + def test_run_return_columns_pysafe_names(self, model): """Addresses https://github.com/JamesPHoughton/pysd/issues/26""" - - model = pysd.read_vensim(test_model) return_columns = ["room_temperature", "teacup_temperature"] result = model.run(return_columns=return_columns) assert set(result.columns) == set(return_columns) - def test_run_output_file(self): - - model = pysd.read_vensim(test_model) - model.progress = False - - error_message = "Paths must be strings or pathlib Path objects." - with pytest.raises(TypeError, match=error_message): - model.run(output_file=1234) - - error_message = "Unsupported output file format .txt" - with pytest.raises(ValueError, match=error_message): - model.run(output_file="file.txt") - - - def test_initial_conditions_invalid(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_initial_conditions_invalid(self, model): error_message = r"Invalid initial conditions\. "\ r"Check documentation for valid entries or use "\ r"'help\(model\.set_initial_condition\)'\." with pytest.raises(TypeError, match=error_message): model.run(initial_condition=["this is not valid"]) - def test_initial_conditions_tuple_pysafe_names(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_initial_conditions_tuple_pysafe_names(self, model): stocks = model.run( initial_condition=(3000, {"teacup_temperature": 33}), return_timestamps=list(range(3000, 3010)) @@ -240,10 +235,9 @@ def test_initial_conditions_tuple_pysafe_names(self): assert stocks["Teacup Temperature"].iloc[0] == 33 - def test_initial_conditions_tuple_original_names(self): + @pytest.mark.parametrize("model_path", [test_model]) + def test_initial_conditions_tuple_original_names(self, model): """ Responds to https://github.com/JamesPHoughton/pysd/issues/77""" - - model = pysd.read_vensim(test_model) stocks = model.run( initial_condition=(3000, {"Teacup Temperature": 33}), return_timestamps=list(range(3000, 3010)), @@ -251,8 +245,8 @@ def test_initial_conditions_tuple_original_names(self): assert stocks.index[0] == 3000 assert stocks["Teacup Temperature"].iloc[0] == 33 - def test_initial_conditions_current(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_initial_conditions_current(self, model): stocks1 = model.run(return_timestamps=list(range(0, 31))) stocks2 = model.run( initial_condition="current", return_timestamps=list(range(30, 45)) @@ -260,27 +254,24 @@ def test_initial_conditions_current(self): assert stocks1["Teacup Temperature"].iloc[-1]\ == stocks2["Teacup Temperature"].iloc[0] - def test_initial_condition_bad_value(self): - model = pysd.read_vensim(test_model) - + @pytest.mark.parametrize("model_path", [test_model]) + def test_initial_condition_bad_value(self, model): with pytest.raises(FileNotFoundError): model.run(initial_condition="bad value") - def test_initial_conditions_subscripted_value_with_numpy_error(self): + @pytest.mark.parametrize("model_path", [test_model_subs]) + def test_initial_conditions_subscripted_value_with_numpy_error(self, + model): input_ = np.array([[5, 3], [4, 8], [9, 3]]) - - model = pysd.read_vensim(test_model_subs) - with pytest.raises(TypeError): model.run(initial_condition=(5, {'stock_a': input_}), return_columns=['stock_a'], return_timestamps=list(range(5, 10))) - def test_set_constant_parameter(self): + @pytest.mark.parametrize("model_path", [test_model]) + def test_set_constant_parameter(self, model): """ In response to: re: https://github.com/JamesPHoughton/pysd/issues/5""" - - model = pysd.read_vensim(test_model) model.set_components({"room_temperature": 20}) assert model.components.room_temperature() == 20 @@ -290,8 +281,8 @@ def test_set_constant_parameter(self): with pytest.raises(NameError): model.set_components({'not_a_var': 20}) - def test_set_constant_parameter_inline(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_set_constant_parameter_inline(self, model): model.components.room_temperature = 20 assert model.components.room_temperature() == 20 @@ -301,8 +292,8 @@ def test_set_constant_parameter_inline(self): with pytest.raises(NameError): model.components.not_a_var = 20 - def test_set_timeseries_parameter(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_set_timeseries_parameter(self, model): timeseries = list(range(30)) temp_timeseries = pd.Series( index=timeseries, @@ -315,8 +306,8 @@ def test_set_timeseries_parameter(self): ) assert (res["room_temperature"] == temp_timeseries).all() - def test_set_timeseries_parameter_inline(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_set_timeseries_parameter_inline(self, model): timeseries = list(range(30)) temp_timeseries = pd.Series( index=timeseries, @@ -329,37 +320,35 @@ def test_set_timeseries_parameter_inline(self): ) assert (res["room_temperature"] == temp_timeseries).all() - def test_set_component_with_real_name(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_set_component_with_real_name(self, model): model.set_components({"Room Temperature": 20}) assert model.components.room_temperature() == 20 model.run(params={"Room Temperature": 70}) assert model.components.room_temperature() == 70 - def test_set_components_warnings(self): + @pytest.mark.parametrize("model_path", [test_model]) + def test_set_components_warnings(self, model): """Addresses https://github.com/JamesPHoughton/pysd/issues/80""" - - model = pysd.read_vensim(test_model) - with catch_warnings(record=True) as w: - simplefilter("always") + warn_message = r"Replacing the equation of stock "\ + r"'Teacup Temperature' with params\.\.\." + with pytest.warns(UserWarning, match=warn_message): model.set_components( {"Teacup Temperature": 20, "Characteristic Time": 15} ) # set stock value using params - # check that warning references the stock - assert "Teacup Temperature" in str(w[0].message) - - def test_set_components_with_function(self): + @pytest.mark.parametrize("model_path", [test_model]) + def test_set_components_with_function(self, model): def test_func(): return 5 - model = pysd.read_vensim(test_model) model.set_components({"Room Temperature": test_func}) res = model.run(return_columns=["Room Temperature"]) assert test_func() == res["Room Temperature"].iloc[0] - def test_set_subscripted_value_with_constant(self): + @pytest.mark.parametrize("model_path", [test_model_subs]) + def test_set_subscripted_value_with_constant(self, model): coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -367,13 +356,13 @@ def test_set_subscripted_value_with_constant(self): dims = ["One Dimensional Subscript", "Second Dimension Subscript"] output = xr.DataArray([[5, 5], [5, 5], [5, 5]], coords, dims) - model = pysd.read_vensim(test_model_subs) model.set_components({"initial_values": 5, "final_time": 10}) res = model.run( return_columns=["Initial Values"], flatten_output=False) assert output.equals(res["Initial Values"].iloc[0]) - def test_set_subscripted_value_with_partial_xarray(self): + @pytest.mark.parametrize("model_path", [test_model_subs]) + def test_set_subscripted_value_with_partial_xarray(self, model): coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -386,13 +375,13 @@ def test_set_subscripted_value_with_partial_xarray(self): ["Second Dimension Subscript"], ) - model = pysd.read_vensim(test_model_subs) model.set_components({"Initial Values": input_val, "final_time": 10}) res = model.run( return_columns=["Initial Values"], flatten_output=False) assert output.equals(res["Initial Values"].iloc[0]) - def test_set_subscripted_value_with_xarray(self): + @pytest.mark.parametrize("model_path", [test_model_subs]) + def test_set_subscripted_value_with_xarray(self, model): coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -400,160 +389,149 @@ def test_set_subscripted_value_with_xarray(self): dims = ["One Dimensional Subscript", "Second Dimension Subscript"] output = xr.DataArray([[5, 3], [4, 8], [9, 3]], coords, dims) - model = pysd.read_vensim(test_model_subs) model.set_components({"initial_values": output, "final_time": 10}) res = model.run( return_columns=["Initial Values"], flatten_output=False) assert output.equals(res["Initial Values"].iloc[0]) - def test_set_parameter_data(self): - model = pysd.read_vensim(test_model_data) + @pytest.mark.parametrize("model_path", [test_model_data]) + @pytest.mark.filterwarnings("ignore") + def test_set_parameter_data(self, model): timeseries = list(range(31)) series = pd.Series( index=timeseries, data=(50+np.random.rand(len(timeseries)).cumsum()) ) - with catch_warnings(): - # avoid warnings related to extrapolation - simplefilter("ignore") - model.set_components({"data_backward": 20, "data_forward": 70}) - - out = model.run( - return_columns=["data_backward", "data_forward"], - flatten_output=False) - - for time in out.index: - assert (out["data_backward"][time] == 20).all() - assert (out["data_forward"][time] == 70).all() - - out = model.run( - return_columns=["data_backward", "data_forward"], - final_time=20, time_step=1, saveper=1, - params={"data_forward": 30, "data_backward": series}, - flatten_output=False) - - for time in out.index: - assert (out["data_forward"][time] == 30).all() - assert (out["data_backward"][time] == series[time]).all() - - def test_set_constant_parameter_lookup(self): - model = pysd.read_vensim(test_model_look) - - with catch_warnings(): - # avoid warnings related to extrapolation - simplefilter("ignore") - model.set_components({"lookup_1d": 20}) - for i in range(100): - assert model.components.lookup_1d(i) == 20 - - model.run(params={"lookup_1d": 70}, final_time=1) - for i in range(100): - assert model.components.lookup_1d(i) == 70 - - model.set_components({"lookup_2d": 20}) - for i in range(100): - assert model.components.lookup_2d(i).equals( - xr.DataArray(20, {"Rows": ["Row1", "Row2"]}, ["Rows"]) - ) + model.set_components({"data_backward": 20, "data_forward": 70}) + + out = model.run( + return_columns=["data_backward", "data_forward"], + flatten_output=False) + + for time in out.index: + assert (out["data_backward"][time] == 20).all() + assert (out["data_forward"][time] == 70).all() + + out = model.run( + return_columns=["data_backward", "data_forward"], + final_time=20, time_step=1, saveper=1, + params={"data_forward": 30, "data_backward": series}, + flatten_output=False) + + for time in out.index: + assert (out["data_forward"][time] == 30).all() + assert (out["data_backward"][time] == series[time]).all() + + @pytest.mark.parametrize("model_path", [test_model_look]) + @pytest.mark.filterwarnings("ignore") + def test_set_constant_parameter_lookup(self, model): + model.set_components({"lookup_1d": 20}) + for i in range(100): + assert model.components.lookup_1d(i) == 20 + + model.run(params={"lookup_1d": 70}, final_time=1) + for i in range(100): + assert model.components.lookup_1d(i) == 70 + + model.set_components({"lookup_2d": 20}) + for i in range(100): + assert model.components.lookup_2d(i).equals( + xr.DataArray(20, {"Rows": ["Row1", "Row2"]}, ["Rows"]) + ) - model.run(params={"lookup_2d": 70}, final_time=1) - for i in range(100): - assert model.components.lookup_2d(i).equals( - xr.DataArray(70, {"Rows": ["Row1", "Row2"]}, ["Rows"]) - ) + model.run(params={"lookup_2d": 70}, final_time=1) + for i in range(100): + assert model.components.lookup_2d(i).equals( + xr.DataArray(70, {"Rows": ["Row1", "Row2"]}, ["Rows"]) + ) - xr1 = xr.DataArray([-10, 50], {"Rows": ["Row1", "Row2"]}, ["Rows"]) - model.set_components({"lookup_2d": xr1}) - for i in range(100): - assert model.components.lookup_2d(i).equals(xr1) + xr1 = xr.DataArray([-10, 50], {"Rows": ["Row1", "Row2"]}, ["Rows"]) + model.set_components({"lookup_2d": xr1}) + for i in range(100): + assert model.components.lookup_2d(i).equals(xr1) - xr2 = xr.DataArray([-100, 500], {"Rows": ["Row1", "Row2"]}, - ["Rows"]) - model.run(params={"lookup_2d": xr2}, final_time=1) - for i in range(100): - assert model.components.lookup_2d(i).equals(xr2) + xr2 = xr.DataArray([-100, 500], {"Rows": ["Row1", "Row2"]}, ["Rows"]) + model.run(params={"lookup_2d": xr2}, final_time=1) + for i in range(100): + assert model.components.lookup_2d(i).equals(xr2) - def test_set_timeseries_parameter_lookup(self): - model = pysd.read_vensim(test_model_look) + @pytest.mark.parametrize("model_path", [test_model_look]) + @pytest.mark.filterwarnings("ignore") + def test_set_timeseries_parameter_lookup(self, model): timeseries = list(range(30)) - with catch_warnings(): - # avoid warnings related to extrapolation - simplefilter("ignore") - temp_timeseries = pd.Series( - index=timeseries, data=(50 + - np.random.rand(len(timeseries) - ).cumsum()) - ) + temp_timeseries = pd.Series( + index=timeseries, + data=(50+np.random.rand(len(timeseries)).cumsum()) + ) - res = model.run( - params={"lookup_1d": temp_timeseries}, - return_columns=["lookup_1d_time"], - return_timestamps=timeseries, - flatten_output=False - ) + res = model.run( + params={"lookup_1d": temp_timeseries}, + return_columns=["lookup_1d_time"], + return_timestamps=timeseries, + flatten_output=False + ) - assert (res["lookup_1d_time"] == temp_timeseries).all() + assert (res["lookup_1d_time"] == temp_timeseries).all() - res = model.run( - params={"lookup_2d": temp_timeseries}, - return_columns=["lookup_2d_time"], - return_timestamps=timeseries, - flatten_output=False - ) + res = model.run( + params={"lookup_2d": temp_timeseries}, + return_columns=["lookup_2d_time"], + return_timestamps=timeseries, + flatten_output=False + ) - assert all( - [ - a.equals(xr.DataArray(b, {"Rows": ["Row1", "Row2"]}, - ["Rows"])) - for a, b in zip(res["lookup_2d_time"].values, - temp_timeseries) - ] - ) + assert all( + [ + a.equals(xr.DataArray(b, {"Rows": ["Row1", "Row2"]}, ["Rows"])) + for a, b in zip(res["lookup_2d_time"].values, + temp_timeseries) + ] + ) - temp_timeseries2 = pd.Series( - index=timeseries, - data=[ - xr.DataArray([50 + x, 20 - y], {"Rows": ["Row1", "Row2"]}, - ["Rows"]) - for x, y in zip( - np.random.rand(len(timeseries)).cumsum(), - np.random.rand(len(timeseries)).cumsum(), - ) - ], - ) + temp_timeseries2 = pd.Series( + index=timeseries, + data=[ + xr.DataArray( + [50 + x, 20 - y], {"Rows": ["Row1", "Row2"]}, ["Rows"] + ) + for x, y in zip( + np.random.rand(len(timeseries)).cumsum(), + np.random.rand(len(timeseries)).cumsum(), + ) + ], + ) - res = model.run( - params={"lookup_2d": temp_timeseries2}, - return_columns=["lookup_2d_time"], - return_timestamps=timeseries, - flatten_output=False - ) + res = model.run( + params={"lookup_2d": temp_timeseries2}, + return_columns=["lookup_2d_time"], + return_timestamps=timeseries, + flatten_output=False + ) - assert all( - [ - a.equals(b) - for a, b in zip(res["lookup_2d_time"].values, - temp_timeseries2) - ] - ) + assert all( + [ + a.equals(b) + for a, b in zip(res["lookup_2d_time"].values, + temp_timeseries2) + ] + ) - def test_set_subscripted_value_with_numpy_error(self): + @pytest.mark.parametrize("model_path", [test_model_subs]) + def test_set_subscripted_value_with_numpy_error(self, model): input_ = np.array([[5, 3], [4, 8], [9, 3]]) - - model = pysd.read_vensim(test_model_subs) with pytest.raises(TypeError): model.set_components({"initial_values": input_, "final_time": 10}) - def test_set_subscripted_timeseries_parameter_with_constant(self): + @pytest.mark.parametrize("model_path", [test_model_subs]) + def test_set_subscripted_timeseries_parameter_with_constant(self, model): coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], } dims = ["One Dimensional Subscript", "Second Dimension Subscript"] - model = pysd.read_vensim(test_model_subs) timeseries = list(range(10)) val_series = [50 + rd for rd in np.random.rand(len(timeseries) ).cumsum()] @@ -574,7 +552,9 @@ def test_set_subscripted_timeseries_parameter_with_constant(self): ] ) - def test_set_subscripted_timeseries_parameter_with_partial_xarray(self): + @pytest.mark.parametrize("model_path", [test_model_subs]) + def test_set_subscripted_timeseries_parameter_with_partial_xarray(self, + model): coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -587,7 +567,6 @@ def test_set_subscripted_timeseries_parameter_with_partial_xarray(self): ["Second Dimension Subscript"], ) - model = pysd.read_vensim(test_model_subs) timeseries = list(range(10)) val_series = [input_val + rd for rd in np.random.rand(len(timeseries) ).cumsum()] @@ -604,7 +583,8 @@ def test_set_subscripted_timeseries_parameter_with_partial_xarray(self): ] ) - def test_set_subscripted_timeseries_parameter_with_xarray(self): + @pytest.mark.parametrize("model_path", [test_model_subs]) + def test_set_subscripted_timeseries_parameter_with_xarray(self, model): coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -613,7 +593,6 @@ def test_set_subscripted_timeseries_parameter_with_xarray(self): init_val = xr.DataArray([[5, 3], [4, 8], [9, 3]], coords, dims) - model = pysd.read_vensim(test_model_subs) timeseries = list(range(10)) temp_timeseries = pd.Series( index=timeseries, @@ -636,10 +615,9 @@ def test_set_subscripted_timeseries_parameter_with_xarray(self): ] ) - def test_docs(self): + @pytest.mark.parametrize("model_path", [test_model]) + def test_docs(self, model): """ Test that the model prints some documentation """ - - model = pysd.read_vensim(test_model) assert isinstance(str(model), str) # tests string conversion of doc = model.doc assert isinstance(doc, pd.DataFrame) @@ -763,8 +741,8 @@ def downstream(run_hist, res_hist): assert run_history == ["U", "D", "D", "D", "U"] assert result_history == ["up", "down", "up", "down", "up", "down"] - def test_initialize(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_initialize(self, model): initial_temp = model.components.teacup_temperature() model.run() final_temp = model.components.teacup_temperature() @@ -773,8 +751,8 @@ def test_initialize(self): assert initial_temp != final_temp assert initial_temp == reset_temp - def test_initialize_order(self): - model = pysd.load(more_tests.joinpath( + def test_initialize_order(self, _root): + model = pysd.load(_root / more_tests.joinpath( "initialization_order/test_initialization_order.py")) assert model.initialize_order == ["_integ_stock_a", "_integ_stock_b"] @@ -785,8 +763,8 @@ def test_initialize_order(self): assert model.components.stock_b() == 1 assert model.components.stock_a() == 1 - def test_set_initial_with_deps(self): - model = pysd.load(more_tests.joinpath("initialization_order/" + def test_set_initial_with_deps(self, _root): + model = pysd.load(_root / more_tests.joinpath("initialization_order/" "test_initialization_order.py")) original_a = model.components.stock_a() @@ -806,9 +784,8 @@ def test_set_initial_with_deps(self): assert model.components.stock_a() == 89 assert model.components.stock_b() == 73 - def test_set_initial_value(self): - model = pysd.read_vensim(test_model) - + @pytest.mark.parametrize("model_path", [test_model]) + def test_set_initial_value(self, model): initial_temp = model.components.teacup_temperature() new_time = np.random.rand() @@ -833,7 +810,8 @@ def test_set_initial_value(self): with pytest.raises(NameError): model.set_initial_value(new_time, {'not_a_var': 500}) - def test_set_initial_value_subscripted_value_with_constant(self): + @pytest.mark.parametrize("model_path", [test_model_subs]) + def test_set_initial_value_subscripted_value_with_constant(self, model): coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -842,8 +820,6 @@ def test_set_initial_value_subscripted_value_with_constant(self): output_b = xr.DataArray([[0, 0], [0, 0], [0, 0]], coords, dims) new_time = np.random.rand() - - model = pysd.read_vensim(test_model_subs) initial_stock = model.components.stock_a() # Test that we can set with real names @@ -866,7 +842,9 @@ def test_set_initial_value_subscripted_value_with_constant(self): {'_integ_stock_a': xr.DataArray(302, {'D': ['A', 'B']}, ['D'])} ) - def test_set_initial_value_subscripted_value_with_partial_xarray(self): + @pytest.mark.parametrize("model_path", [test_model_subs]) + def test_set_initial_value_subscripted_value_with_partial_xarray(self, + model): coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -895,7 +873,6 @@ def test_set_initial_value_subscripted_value_with_partial_xarray(self): new_time = np.random.rand() - model = pysd.read_vensim(test_model_subs) initial_stock = model.components.stock_a() # Test that we can set with real names @@ -911,7 +888,8 @@ def test_set_initial_value_subscripted_value_with_partial_xarray(self): model.set_initial_value(new_time + 2, {'_integ_stock_a': input_val3}) assert model.components.stock_a().equals(output3) - def test_set_initial_value_subscripted_value_with_xarray(self): + @pytest.mark.parametrize("model_path", [test_model_subs]) + def test_set_initial_value_subscripted_value_with_xarray(self, model): coords = { "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], "Second Dimension Subscript": ["Column 1", "Column 2"], @@ -923,7 +901,6 @@ def test_set_initial_value_subscripted_value_with_xarray(self): new_time = np.random.rand() - model = pysd.read_vensim(test_model_subs) initial_stock = model.components.stock_a() # Test that we can set with real names @@ -939,15 +916,14 @@ def test_set_initial_value_subscripted_value_with_xarray(self): model.set_initial_value(new_time + 2, {'_integ_stock_a': output3}) assert model.components.stock_a().equals(output3) - def test_set_initial_value_subscripted_value_with_numpy_error(self): + @pytest.mark.parametrize("model_path", [test_model_subs]) + def test_set_initial_value_subscripted_value_with_numpy_error(self, model): input1 = np.array([[5, 3], [4, 8], [9, 3]]) input2 = np.array([[53, 43], [84, 80], [29, 63]]) input3 = np.array([[54, 32], [40, 87], [93, 93]]) new_time = np.random.rand() - model = pysd.read_vensim(test_model_subs) - # Test that we can set with real names with pytest.raises(TypeError): model.set_initial_value(new_time, {'Stock A': input1}) @@ -960,16 +936,16 @@ def test_set_initial_value_subscripted_value_with_numpy_error(self): with pytest.raises(TypeError): model.set_initial_value(new_time + 2, {'_integ_stock_a': input3}) - def test_replace_element(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_replace_element(self, model): stocks1 = model.run() model.components.characteristic_time = lambda: 3 stocks2 = model.run() assert stocks1["Teacup Temperature"].loc[10]\ > stocks2["Teacup Temperature"].loc[10] - def test_set_initial_condition_origin_full(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_set_initial_condition_origin_full(self, model): initial_temp = model.components.teacup_temperature() initial_time = model.components.time() @@ -996,8 +972,8 @@ def test_set_initial_condition_origin_full(self): assert initial_temp == set_temp assert initial_time == set_time - def test_set_initial_condition_origin_short(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_set_initial_condition_origin_short(self, model): initial_temp = model.components.teacup_temperature() initial_time = model.components.time() @@ -1024,8 +1000,8 @@ def test_set_initial_condition_origin_short(self): assert initial_temp == set_temp assert initial_time == set_time - def test_set_initial_condition_for_stock_component(self): - model = pysd.read_vensim(test_model) + @pytest.mark.parametrize("model_path", [test_model]) + def test_set_initial_condition_for_stock_component(self, model): initial_temp = model.components.teacup_temperature() initial_time = model.components.time() @@ -1046,9 +1022,8 @@ def test_set_initial_condition_for_stock_component(self): assert set_time == 10 - def test_set_initial_condition_for_constant_component(self): - model = pysd.read_vensim(test_model) - + @pytest.mark.parametrize("model_path", [test_model]) + def test_set_initial_condition_for_constant_component(self, model): new_state = {"Room Temperature": 100} new_time = 10 @@ -1057,53 +1032,91 @@ def test_set_initial_condition_for_constant_component(self): with pytest.raises(ValueError, match=error_message): model.set_initial_condition((new_time, new_state)) - def test_get_args(self): - model = pysd.read_vensim(test_model) - model2 = pysd.read_vensim(test_model_look) - - assert model.get_args('Room Temperature') == [] - assert model.get_args('room_temperature') == [] - assert model.get_args('teacup_temperature') == [] - assert model.get_args('_integ_teacup_temperature') == [] - - assert model2.get_args('lookup 1d') == ['x', 'final_subs'] - assert model2.get_args('lookup_1d') == ['x', 'final_subs'] - assert model2.get_args('lookup 2d') == ['x', 'final_subs'] - assert model2.get_args('lookup_2d') == ['x', 'final_subs'] + @pytest.mark.parametrize( + "model_path,args", + [ + ( + test_model, + { + "Room Temperature": [], + "room_temperature": [], + "teacup_temperature": [], + "_integ_teacup_temperature": [] + } + ), + ( + test_model_look, + { + "lookup 1d": ["x", "final_subs"], + "lookup_1d": ["x", "final_subs"], + "lookup 2d": ["x", "final_subs"], + "lookup_2d": ["x", "final_subs"], + } + ) + ]) + def test_get_args(self, model, args): + for var, arg in args.items(): + assert model.get_args(var) == arg with pytest.raises(NameError): - model.get_args('not_a_var') - - def test_get_coords(self): - coords = { - "One Dimensional Subscript": ["Entry 1", "Entry 2", "Entry 3"], - "Second Dimension Subscript": ["Column 1", "Column 2"], - } - dims = ["One Dimensional Subscript", "Second Dimension Subscript"] - - coords_dims = (coords, dims) - - model = pysd.read_vensim(test_model) - model2 = pysd.read_vensim(test_model_subs) - - assert model.get_coords("Room Temperature") is None - assert model.get_coords("room_temperature") is None - assert model.get_coords("teacup_temperature") is None - assert model.get_coords("_integ_teacup_temperature") is None - - assert model2.get_coords("Initial Values") == coords_dims - assert model2.get_coords("initial_values") == coords_dims - assert model2.get_coords("Stock A") == coords_dims - assert model2.get_coords("stock_a") == coords_dims - assert model2.get_coords("_integ_stock_a") == coords_dims + model.get_args("not_a_var") + + @pytest.mark.parametrize( + "model_path,coords", + [ + ( + test_model, + { + "Room Temperature": None, + "room_temperature": None, + "teacup_temperature": None, + "_integ_teacup_temperature": None + } + ), + ( + test_model_subs, + { + "Initial Values": { + "One Dimensional Subscript": ["Entry 1", "Entry 2", + "Entry 3"], + "Second Dimension Subscript":["Column 1", "Column 2"] + }, + "initial_values": { + "One Dimensional Subscript": ["Entry 1", "Entry 2", + "Entry 3"], + "Second Dimension Subscript":["Column 1", "Column 2"] + }, + "Stock A": { + "One Dimensional Subscript": ["Entry 1", "Entry 2", + "Entry 3"], + "Second Dimension Subscript":["Column 1", "Column 2"] + }, + "stock_a": { + "One Dimensional Subscript": ["Entry 1", "Entry 2", + "Entry 3"], + "Second Dimension Subscript":["Column 1", "Column 2"] + }, + "_integ_stock_a": { + "One Dimensional Subscript": ["Entry 1", "Entry 2", + "Entry 3"], + "Second Dimension Subscript":["Column 1", "Column 2"] + } + } + ) + ]) + def test_get_coords(self, model, coords): + for var, coord in coords.items(): + if coord is not None: + coord = coord, list(coord) + assert model.get_coords(var) == coord with pytest.raises(NameError): - model.get_coords('not_a_var') + model.get_coords("not_a_var") - def test_getitem(self): - model = pysd.read_vensim(test_model) - model2 = pysd.read_vensim(test_model_look) - model3 = pysd.read_vensim(test_model_data) + def test_getitem(self, _root): + model = pysd.read_vensim(_root / test_model) + model2 = pysd.read_vensim(_root / test_model_look) + model3 = pysd.read_vensim(_root / test_model_data) coords = {'Dim': ['A', 'B'], 'Rows': ['Row1', 'Row2']} room_temp = 70 @@ -1133,10 +1146,10 @@ def test_getitem(self): model3.run() assert model3['data backward'].equals(data1) - def test_get_series_data(self): - model = pysd.read_vensim(test_model) - model2 = pysd.read_vensim(test_model_look) - model3 = pysd.read_vensim(test_model_data) + def test_get_series_data(self, _root): + model = pysd.read_vensim(_root / test_model) + model2 = pysd.read_vensim(_root / test_model_look) + model3 = pysd.read_vensim(_root / test_model_data) error_message = "Trying to get the values of a constant variable." with pytest.raises(ValueError, match=error_message): @@ -1194,11 +1207,10 @@ def test_get_series_data(self): data = model3.get_series_data('_ext_data_data_backward') assert data.equals(data_exp) - def test__integrate(self, shared_tmpdir): + @pytest.mark.parametrize("model_path", [test_model]) + def test__integrate(self, tmp_path, model): from pysd.py_backend.model import ModelOutput # TODO: think through a stronger test here... - model = pysd.read_vensim(test_model) - model.progress = False model.time.add_return_timestamps(list(range(0, 5, 2))) capture_elements = {'teacup_temperature'} @@ -1209,22 +1221,19 @@ def test__integrate(self, shared_tmpdir): assert 'teacup_temperature' in res assert all(res.index.values == list(range(0, 5, 2))) - model = pysd.read_vensim(test_model) - model.progress = False + model.reload() model.time.add_return_timestamps(list(range(0, 5, 2))) out = ModelOutput(model, capture_elements, - shared_tmpdir.joinpath("output.nc")) + tmp_path.joinpath("output.nc")) model._integrate(out) res = out.handler.ds assert isinstance(res, nc.Dataset) assert 'teacup_temperature' in res.variables - with catch_warnings(record=True) as w: - simplefilter("always") - assert np.array_equal(res["time"][:].data, np.arange(0, 5, 2)) + assert np.array_equal(res["time"][:].data, np.arange(0, 5, 2)) res.close() - def test_default_returns_with_construction_functions(self): + def test_default_returns_with_construction_functions(self, _root): """ If the run function is called with no arguments, should still be able to get default return functions. @@ -1244,37 +1253,37 @@ def test_default_returns_with_construction_functions(self): "Output Delay3", } <= set(ret.columns.values) - def test_default_returns_with_lookups(self): + @pytest.mark.parametrize( + "model_path", + [Path("test-models/tests/lookups/test_lookups.mdl")]) + def test_default_returns_with_lookups(self, model): """ Addresses https://github.com/JamesPHoughton/pysd/issues/114 The default settings should skip model elements with no particular return value """ - - model = pysd.read_vensim( - _root.joinpath("test-models/tests/lookups/test_lookups.mdl")) ret = model.run() assert {"accumulation", "rate", "lookup function call"}\ <= set(ret.columns.values) - def test_py_model_file(self): + @pytest.mark.parametrize("model_path", [test_model]) + def test_files(self, model, model_path, tmp_path): """Addresses https://github.com/JamesPHoughton/pysd/issues/86""" - model = pysd.read_vensim(test_model) - assert model.py_model_file == str(test_model.with_suffix(".py")) - - def test_mdl_file(self): - """Relates to https://github.com/JamesPHoughton/pysd/issues/86""" + # Path from where the model is translated + path = tmp_path / model_path.parent.name / model_path.name - model = pysd.read_vensim(test_model) - assert model.mdl_file == str(test_model) + # Check py_model_file + assert model.py_model_file == str(path.with_suffix(".py")) + # Check mdl_file + assert model.mdl_file == str(path) class TestModelInteraction(): """ The tests in this class test pysd's interaction with itself and other modules. """ - def test_multiple_load(self): + def test_multiple_load(self, _root): """ Test that we can load and run multiple models at the same time, and that the models don't interact with each other. This can @@ -1297,7 +1306,7 @@ def test_multiple_load(self): assert "susceptible" not in dir(model_1.components) assert "teacup_temperature" in dir(model_1.components) - def test_no_crosstalk(self): + def test_no_crosstalk(self, _root): """ Need to check that if we instantiate two copies of the same model, changes to one copy do not influence the other copy. @@ -1320,14 +1329,14 @@ def test_no_crosstalk(self): model_1.run() assert model_1.time() != model_2.time() - def test_restart_cache(self): + @pytest.mark.parametrize("model_path", [test_model]) + def test_restart_cache(self, model): """ Test that when we cache a model variable at the 'run' time, - if the variable is changed and the model re-run, the cache updates - to the new variable, instead of maintaining the old one. - """ + if the variable is changed and the model re-run, the cache updates + to the new variable, instead of maintaining the old one. - model = pysd.read_vensim(test_model) + """ model.run() old = model.components.room_temperature() model.set_components({"Room Temperature": 345}) @@ -1351,7 +1360,7 @@ def test_not_able_to_update_stateful_object(self): class TestMultiRun(): - def test_delay_reinitializes(self): + def test_delay_reinitializes(self, _root): model = pysd.read_vensim(_root.joinpath( "test-models/tests/delays/test_delays.mdl")) res1 = model.run() @@ -1360,93 +1369,85 @@ def test_delay_reinitializes(self): class TestDependencies(): - def test_teacup_deps(self): - from pysd import read_vensim - - model = read_vensim(test_model) - - expected_dep = { - 'characteristic_time': {}, - 'heat_loss_to_room': { - 'teacup_temperature': 1, - 'room_temperature': 1, - 'characteristic_time': 1 - }, - 'room_temperature': {}, - 'teacup_temperature': {'_integ_teacup_temperature': 1}, - '_integ_teacup_temperature': { - 'initial': {}, - 'step': {'heat_loss_to_room': 1} - }, - 'final_time': {}, - 'initial_time': {}, - 'saveper': {'time_step': 1}, - 'time_step': {} - } - assert model.dependencies == expected_dep - def test_multiple_deps(self): - from pysd import read_vensim - - model = read_vensim( - more_tests.joinpath( - "subscript_individually_defined_stocks2/" - + "test_subscript_individually_defined_stocks2.mdl")) - - expected_dep = { - "stock_a": {"_integ_stock_a": 1, "_integ_stock_a_1": 1}, - "inflow_a": {"rate_a": 1}, - "inflow_b": {"rate_a": 1}, - "initial_values": {"initial_values_a": 1, "initial_values_b": 1}, - "initial_values_a": {}, - "initial_values_b": {}, - "rate_a": {}, - "final_time": {}, - "initial_time": {}, - "saveper": {"time_step": 1}, - "time_step": {}, - "_integ_stock_a": { - "initial": {"initial_values": 1}, - "step": {"inflow_a": 1} - }, - '_integ_stock_a_1': { - 'initial': {'initial_values': 1}, - 'step': {'inflow_b': 1} - } - } - assert model.dependencies == expected_dep - - more_tests.joinpath( - "subscript_individually_defined_stocks2/" - + "test_subscript_individually_defined_stocks2.py").unlink() - - def test_constant_deps(self): - from pysd import read_vensim - - model = read_vensim(test_model_constant_pipe) - - expected_dep = { - "constant1": {}, - "constant2": {"constant1": 1}, - "constant3": {"constant1": 3, "constant2": 1}, - "final_time": {}, - "initial_time": {}, - "time_step": {}, - "saveper": {"time_step": 1} - } + @pytest.mark.parametrize( + "model_path,expected_dep", + [ + ( + test_model, + { + 'characteristic_time': {}, + 'heat_loss_to_room': { + 'teacup_temperature': 1, + 'room_temperature': 1, + 'characteristic_time': 1 + }, + 'room_temperature': {}, + 'teacup_temperature': {'_integ_teacup_temperature': 1}, + '_integ_teacup_temperature': { + 'initial': {}, + 'step': {'heat_loss_to_room': 1} + }, + 'final_time': {}, + 'initial_time': {}, + 'saveper': {'time_step': 1}, + 'time_step': {} + } + + ), + ( + more_tests.joinpath( + "subscript_individually_defined_stocks2/" + "test_subscript_individually_defined_stocks2.mdl"), + { + "stock_a": {"_integ_stock_a": 1, "_integ_stock_a_1": 1}, + "inflow_a": {"rate_a": 1}, + "inflow_b": {"rate_a": 1}, + "initial_values": { + "initial_values_a": 1, + "initial_values_b": 1 + }, + "initial_values_a": {}, + "initial_values_b": {}, + "rate_a": {}, + "final_time": {}, + "initial_time": {}, + "saveper": {"time_step": 1}, + "time_step": {}, + "_integ_stock_a": { + "initial": {"initial_values": 1}, + "step": {"inflow_a": 1} + }, + '_integ_stock_a_1': { + 'initial': {'initial_values': 1}, + 'step': {'inflow_b': 1} + } + } + ), + ( + test_model_constant_pipe, + { + "constant1": {}, + "constant2": {"constant1": 1}, + "constant3": {"constant1": 3, "constant2": 1}, + "final_time": {}, + "initial_time": {}, + "time_step": {}, + "saveper": {"time_step": 1} + } + ) + ], + ids=["teacup", "multiple", "constant"]) + def test_deps(self, model, expected_dep, model_path): assert model.dependencies == expected_dep - for key, value in model.cache_type.items(): - if key != "time": - assert value == "run" - - test_model_constant_pipe.with_suffix(".py").unlink() - - def test_change_constant_pipe(self): - from pysd import read_vensim - - model = read_vensim(test_model_constant_pipe) + if model_path == test_model_constant_pipe: + for key, value in model.cache_type.items(): + if key != "time": + assert value == "run" + @pytest.mark.parametrize("model_path", [test_model_constant_pipe]) + def test_change_constant_pipe(self, model): new_var = pd.Series( index=[0, 1, 2, 3, 4, 5], data=[1, 2, 3, 4, 5, 6]) @@ -1473,212 +1474,98 @@ def test_change_constant_pipe(self): assert\ (out2["constant3"] == (5*new_var.values-1)*new_var.values).all() - test_model_constant_pipe.with_suffix(".py").unlink() - class TestExportImport(): - def test_run_export_import_integ(self): - from pysd import read_vensim - - with catch_warnings(): - simplefilter("ignore") - model = read_vensim(test_model) - stocks = model.run(return_timestamps=[0, 10, 20, 30]) - assert (stocks['INITIAL TIME'] == 0).all().all() - assert (stocks['FINAL TIME'] == 30).all().all() - - model.reload() - stocks1 = model.run(return_timestamps=[0, 10], final_time=12) - assert (stocks1['INITIAL TIME'] == 0).all().all() - assert (stocks1['FINAL TIME'] == 12).all().all() - model.export('teacup12.pic') - model.reload() - stocks2 = model.run(initial_condition='teacup12.pic', - return_timestamps=[20, 30]) - assert (stocks2['INITIAL TIME'] == 12).all().all() - assert (stocks2['FINAL TIME'] == 30).all().all() - stocks.drop('INITIAL TIME', axis=1, inplace=True) - stocks1.drop('INITIAL TIME', axis=1, inplace=True) - stocks2.drop('INITIAL TIME', axis=1, inplace=True) - stocks.drop('FINAL TIME', axis=1, inplace=True) - stocks1.drop('FINAL TIME', axis=1, inplace=True) - stocks2.drop('FINAL TIME', axis=1, inplace=True) - Path('teacup12.pic').unlink() - - assert_frames_close(stocks1, stocks.loc[[0, 10]]) - assert_frames_close(stocks2, stocks.loc[[20, 30]]) - - def test_run_export_import_delay(self): - from pysd import read_vensim - - with catch_warnings(): - simplefilter("ignore") - test_delays = _root.joinpath( - 'test-models/tests/delays/test_delays.mdl') - model = read_vensim(test_delays) - stocks = model.run(return_timestamps=20) - model.reload() - model.run(return_timestamps=[], final_time=7) - model.export('delays7.pic') - stocks2 = model.run(initial_condition='delays7.pic', - return_timestamps=20) - assert (stocks['INITIAL TIME'] == 0).all().all() - assert (stocks2['INITIAL TIME'] == 7).all().all() - stocks.drop('INITIAL TIME', axis=1, inplace=True) - stocks2.drop('INITIAL TIME', axis=1, inplace=True) - stocks.drop('FINAL TIME', axis=1, inplace=True) - stocks2.drop('FINAL TIME', axis=1, inplace=True) - Path('delays7.pic').unlink() - - assert_frames_close(stocks2, stocks) - - def test_run_export_import_delay_fixed(self): - from pysd import read_vensim - - with catch_warnings(): - simplefilter("ignore") - test_delayf = _root.joinpath( - 'test-models/tests/delay_fixed/test_delay_fixed.mdl') - model = read_vensim(test_delayf) - stocks = model.run(return_timestamps=20) - model.reload() - model.run(return_timestamps=7) - model.export('delayf7.pic') - stocks2 = model.run(initial_condition='delayf7.pic', - return_timestamps=20) - assert (stocks['INITIAL TIME'] == 0).all().all() - assert (stocks2['INITIAL TIME'] == 7).all().all() - stocks.drop('INITIAL TIME', axis=1, inplace=True) - stocks2.drop('INITIAL TIME', axis=1, inplace=True) - stocks.drop('FINAL TIME', axis=1, inplace=True) - stocks2.drop('FINAL TIME', axis=1, inplace=True) - Path('delayf7.pic').unlink() - - assert_frames_close(stocks2, stocks) - - def test_run_export_import_forecast(self): - from pysd import read_vensim - - with catch_warnings(): - simplefilter("ignore") - test_trend = _root.joinpath( - 'test-models/tests/forecast/' - + 'test_forecast.mdl') - model = read_vensim(test_trend) - stocks = model.run(return_timestamps=50, flatten_output=True) - model.reload() - model.run(return_timestamps=20) - model.export('frcst20.pic') - stocks2 = model.run(initial_condition='frcst20.pic', - return_timestamps=50, - flatten_output=True) - assert (stocks['INITIAL TIME'] == 0).all().all() - assert (stocks2['INITIAL TIME'] == 20).all().all() - stocks.drop('INITIAL TIME', axis=1, inplace=True) - stocks2.drop('INITIAL TIME', axis=1, inplace=True) - stocks.drop('FINAL TIME', axis=1, inplace=True) - stocks2.drop('FINAL TIME', axis=1, inplace=True) - Path('frcst20.pic').unlink() - - assert_frames_close(stocks2, stocks) - - def test_run_export_import_sample_if_true(self): - from pysd import read_vensim - - with catch_warnings(): - simplefilter("ignore") - test_sample_if_true = _root.joinpath( - 'test-models/tests/sample_if_true/test_sample_if_true.mdl') - model = read_vensim(test_sample_if_true) - stocks = model.run(return_timestamps=20, flatten_output=True) - model.reload() - model.run(return_timestamps=7) - model.export('sample_if_true7.pic') - stocks2 = model.run(initial_condition='sample_if_true7.pic', - return_timestamps=20, - flatten_output=True) - assert (stocks['INITIAL TIME'] == 0).all().all() - assert (stocks2['INITIAL TIME'] == 7).all().all() - stocks.drop('INITIAL TIME', axis=1, inplace=True) - stocks2.drop('INITIAL TIME', axis=1, inplace=True) - stocks.drop('FINAL TIME', axis=1, inplace=True) - stocks2.drop('FINAL TIME', axis=1, inplace=True) - Path('sample_if_true7.pic').unlink() - - assert_frames_close(stocks2, stocks) - - def test_run_export_import_smooth(self): - from pysd import read_vensim - - with catch_warnings(): - simplefilter("ignore") - test_smooth = _root.joinpath( - 'test-models/tests/subscripted_smooth/' - + 'test_subscripted_smooth.mdl') - model = read_vensim(test_smooth) - stocks = model.run(return_timestamps=20, flatten_output=True) - model.reload() - model.run(return_timestamps=7) - model.export('smooth7.pic') - stocks2 = model.run(initial_condition='smooth7.pic', - return_timestamps=20, - flatten_output=True) - assert (stocks['INITIAL TIME'] == 0).all().all() - assert (stocks2['INITIAL TIME'] == 7).all().all() - stocks.drop('INITIAL TIME', axis=1, inplace=True) - stocks2.drop('INITIAL TIME', axis=1, inplace=True) - stocks.drop('FINAL TIME', axis=1, inplace=True) - stocks2.drop('FINAL TIME', axis=1, inplace=True) - Path('smooth7.pic').unlink() - - assert_frames_close(stocks2, stocks) - - def test_run_export_import_trend(self): - from pysd import read_vensim - - with catch_warnings(): - simplefilter("ignore") - test_trend = _root.joinpath( - 'test-models/tests/subscripted_trend/' - + 'test_subscripted_trend.mdl') - model = read_vensim(test_trend) - stocks = model.run(return_timestamps=20, flatten_output=True) - model.reload() - model.run(return_timestamps=7) - model.export('trend7.pic') - stocks2 = model.run(initial_condition='trend7.pic', - return_timestamps=20, - flatten_output=True) - assert (stocks['INITIAL TIME'] == 0).all().all() - assert (stocks2['INITIAL TIME'] == 7).all().all() - stocks.drop('INITIAL TIME', axis=1, inplace=True) - stocks2.drop('INITIAL TIME', axis=1, inplace=True) - stocks.drop('FINAL TIME', axis=1, inplace=True) - stocks2.drop('FINAL TIME', axis=1, inplace=True) - Path('trend7.pic').unlink() - - assert_frames_close(stocks2, stocks) - - def test_run_export_import_initial(self): - from pysd import read_vensim - - with catch_warnings(): - simplefilter("ignore") - test_initial = _root.joinpath( - 'test-models/tests/initial_function/test_initial.mdl') - model = read_vensim(test_initial) - stocks = model.run(return_timestamps=20) - model.reload() - model.run(return_timestamps=7) - model.export('initial7.pic') - stocks2 = model.run(initial_condition='initial7.pic', - return_timestamps=20) - assert (stocks['INITIAL TIME'] == 0).all().all() - assert (stocks2['INITIAL TIME'] == 7).all().all() - stocks.drop('INITIAL TIME', axis=1, inplace=True) - stocks2.drop('INITIAL TIME', axis=1, inplace=True) - stocks.drop('FINAL TIME', axis=1, inplace=True) - stocks2.drop('FINAL TIME', axis=1, inplace=True) - Path('initial7.pic').unlink() - - assert_frames_close(stocks2, stocks) + + @pytest.mark.parametrize( + "model_path,return_ts,final_t", + [ + ( + test_model, + ([0, 10, 20, 30], [0, 10], [20, 30]), + (None, 12, None) + ), + ( + Path('test-models/tests/delays/test_delays.mdl'), + ([10, 20], [], [10, 20]), + (None, 7, 34) + ), + ( + Path('test-models/tests/delay_fixed/test_delay_fixed.mdl'), + ([7, 20], [7], [20]), + (None, None, None) + ), + ( + Path('test-models/tests/forecast/test_forecast.mdl'), + ([20, 30, 50], [20, 30], [50]), + (55, 32, 52) + ), + ( + Path( + 'test-models/tests/sample_if_true/test_sample_if_true.mdl' + ), + ([8, 20], [8], [20]), + (None, 15, None) + ), + ( + Path('test-models/tests/subscripted_smooth/' + 'test_subscripted_smooth.mdl'), + ([8, 20], [8], [20]), + (None, 15, None) + ), + ( + Path('test-models/tests/subscripted_trend/' + 'test_subscripted_trend.mdl'), + ([8, 20], [8], [20]), + (None, 15, None) + + ), + ( + Path('test-models/tests/initial_function/test_initial.mdl'), + ([8, 20], [8], [20]), + (None, 15, None) + ) + ], + ids=["integ", "delays", "delay_fixed", "forecast", "sample_if_true", + "smooth", "trend", "initial"] + ) + @pytest.mark.filterwarnings("ignore") + def test_run_export_import(self, tmp_path, model, return_ts, final_t): + export_path = tmp_path / "export.pic" + + # Final times of each run + finals = [final_t[i] or return_ts[i][-1] for i in range(3)] + + # Whole run + stocks = model.run( + return_timestamps=return_ts[0], final_time=final_t[0] + ) + assert (stocks['INITIAL TIME'] == 0).all().all() + assert (stocks['FINAL TIME'] == finals[0]).all().all() + + # Export run + model.reload() + stocks1 = model.run( + return_timestamps=return_ts[1], final_time=final_t[1] + ) + assert (stocks1['INITIAL TIME'] == 0).all().all() + assert (stocks1['FINAL TIME'] == finals[1]).all().all() + model.export(export_path) + + # Import run + model.reload() + stocks2 = model.run( + initial_condition=export_path, + return_timestamps=return_ts[2], final_time=final_t[2] + ) + assert (stocks2['INITIAL TIME'] == finals[1]).all().all() + assert (stocks2['FINAL TIME'] == finals[2]).all().all() + + # Compare results + stocks.drop(columns=["INITIAL TIME", "FINAL TIME"], inplace=True) + stocks1.drop(columns=["INITIAL TIME", "FINAL TIME"], inplace=True) + stocks2.drop(columns=["INITIAL TIME", "FINAL TIME"], inplace=True) + if return_ts[1]: + assert_frames_close(stocks1, stocks.loc[return_ts[1]]) + if return_ts[2]: + assert_frames_close(stocks2, stocks.loc[return_ts[2]]) From a8519344facb496be191003f618b92f782ab8576 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez Date: Fri, 16 Sep 2022 16:47:12 +0200 Subject: [PATCH 42/43] Filter warnings --- tests/pytest.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytest.ini b/tests/pytest.ini index b7a740f8..b3e76fa3 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -3,4 +3,5 @@ python_files = pytest_*/**/*.py pytest_*/*.py filterwarnings = error ignore:Creating an ndarray from ragged nested sequences - ignore:distutils Version classes are deprecated. Use packaging.version instead \ No newline at end of file + ignore:distutils Version classes are deprecated. Use packaging.version instead + ignore:`np.bool` is a deprecated alias for the builtin `bool`. To silence this warning, use `bool` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.bool_` here. From afc8702480f557c2c21b52655035c94dc7a3fca8 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez <61285767+enekomartinmartinez@users.noreply.github.com> Date: Mon, 19 Sep 2022 17:14:16 +0200 Subject: [PATCH 43/43] migrate to SDXorg (#362) * migrate to SDXorg * Update crashing links * Add cookbook note in getting_started * Update whats_new.rst * reduce relative precision * Add convergence tests * Remove contributing * Add building docs workflow * Update about * Fix docstring * minor fixes * Remove old deprecation warning --- .github/workflows/build-docs.yml | 27 +++ .github/workflows/link-check.yml | 18 ++ CONTRIBUTING.md | 3 - LICENSE | 2 +- MANIFEST.in | 2 +- README.md | 31 ++- docs/about.rst | 17 +- docs/conf.py | 10 +- docs/development/guidelines.rst | 2 +- docs/development/pathway.rst | 2 +- docs/generate_tables.py | 3 +- docs/getting_started.rst | 8 + docs/index.rst | 23 +- docs/installation.rst | 6 +- docs/reporting_bugs.rst | 2 +- docs/requirements.txt | 2 + docs/whats_new.rst | 144 ++++++------ pysd/py_backend/components.py | 6 +- pysd/pysd.py | 2 +- setup.py | 5 +- tests/pytest.ini | 1 - .../pytest_integration_euler.py | 211 ++++++++++++++++++ tests/pytest_pysd/pytest_pysd.py | 23 +- tests/test-models | 2 +- 24 files changed, 422 insertions(+), 130 deletions(-) create mode 100644 .github/workflows/build-docs.yml create mode 100644 .github/workflows/link-check.yml delete mode 100644 CONTRIBUTING.md create mode 100644 tests/pytest_integration/pytest_integration_euler.py diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml new file mode 100644 index 00000000..e425613b --- /dev/null +++ b/.github/workflows/build-docs.yml @@ -0,0 +1,27 @@ + +# Build html docs. Any warning building the docs will produce an error. + +name: Build docs + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.7' + - name: Install dependencies + run: | + pip install . + pip install -r docs/requirements.txt + - name: Test build html + run: | + cd docs + make html SPHINXOPTS="-W" diff --git a/.github/workflows/link-check.yml b/.github/workflows/link-check.yml new file mode 100644 index 00000000..62174758 --- /dev/null +++ b/.github/workflows/link-check.yml @@ -0,0 +1,18 @@ +name: Link check + +on: + push: + pull_request: + +jobs: + linkChecker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Link Checker + uses: lycheeverse/lychee-action@v1.5.1 + with: + fail: true + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 6cb92be8..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,3 +0,0 @@ -..under construction.. - -We are super welcoming of contributions, here's how: diff --git a/LICENSE b/LICENSE index 13c0d569..d4977050 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013-2017 James Houghton +Copyright (c) 2013-2022 PySD contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/MANIFEST.in b/MANIFEST.in index 248b7e48..d1c65b15 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include requirements.txt include README.md -include docs/images/PySD_Logo* include LICENSE +include docs/images/PySD_Logo* graft pysd/translators/*/parsing_grammars diff --git a/README.md b/README.md index 444df8bc..27428f2f 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,18 @@ PySD ==== - -[![Coverage Status](https://coveralls.io/repos/github/JamesPHoughton/pysd/badge.svg?branch=master)](https://coveralls.io/github/JamesPHoughton/pysd?branch=master) +[![Maintained](https://img.shields.io/badge/Maintained-Yes-brightgreen.svg)](https://github.com/SDXorg/pysd/pulse) +[![Coverage Status](https://coveralls.io/repos/github/SDXorg/pysd/badge.svg?branch=master)](https://coveralls.io/github/SDXorg/pysd?branch=master) [![Anaconda-Server Badge](https://anaconda.org/conda-forge/pysd/badges/version.svg)](https://anaconda.org/conda-forge/pysd) [![PyPI version](https://badge.fury.io/py/pysd.svg)](https://badge.fury.io/py/pysd) [![PyPI status](https://img.shields.io/pypi/status/pysd.svg)](https://pypi.python.org/pypi/pysd/) [![Py version](https://img.shields.io/pypi/pyversions/pysd.svg)](https://pypi.python.org/pypi/pysd/) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5654824.svg)](https://doi.org/10.5281/zenodo.5654824) +[![Contributions](https://img.shields.io/badge/contributions-welcome-blue.svg)](https://pysd.readthedocs.io/en/latest/development/development_index.html) [![Docs](https://readthedocs.org/projects/pysd/badge/?version=latest)](https://pysd.readthedocs.io/en/latest/?badge=latest) -![PySD Logo](https://raw.githubusercontent.com/JamesPHoughton/pysd/5cc4fe5dc65e6b5140a00e87a1be9d261570ee8d/docs/images/PySD_Logo_letters.svg?style=centerme) +![PySD Logo](https://raw.githubusercontent.com/SDXorg/pysd/5cc4fe5dc65e6b5140a00e87a1be9d261570ee8d/docs/images/PySD_Logo_letters.svg?style=centerme) -This project is a simple library for running [System Dynamics](http://en.wikipedia.org/wiki/System_dynamics) models in python, with the purpose of improving integration of *Big Data* and *Machine Learning* into the SD workflow. +This project is a library for running [System Dynamics](http://en.wikipedia.org/wiki/System_dynamics) models in Python, with the purpose of improving integration of *Big Data* and *Machine Learning* into the SD workflow. **The current version needs to run at least Python 3.7.** @@ -22,13 +23,13 @@ See the [project documentation](http://pysd.readthedocs.org/) for information ab - [Installation](http://pysd.readthedocs.org/en/latest/installation.html) - [Getting Started](http://pysd.readthedocs.org/en/latest/getting_started.html) -For standard methods for data analysis with SD models, see the [PySD Cookbook](https://github.com/JamesPHoughton/PySD-Cookbook), containing (for example): +For standard methods for data analysis with SD models, see the [PySD Cookbook](https://github.com/SDXorg/PySD-Cookbook), containing (for example): -- [Model Fitting](http://nbviewer.ipython.org/github/JamesPHoughton/PySD-Cookbook/blob/master/2_1_Fitting_with_Optimization.ipynb) -- [Surrogating model components with machine learning regressions](http://nbviewer.ipython.org/github/JamesPHoughton/PySD-Cookbook/blob/master/6_1_Surrogating_with_regression.ipynb) -- [Multi-Scale geographic comparison of model predictions](http://nbviewer.ipython.org/github/JamesPHoughton/PySD-Cookbook/blob/master/Exploring%20models%20across%20geographic%20scales.ipynb) +- [Model Fitting](http://nbviewer.ipython.org/github/SDXorg/PySD-Cookbook/blob/master/source/analyses/fitting/Fitting_with_Optimization.ipynb) +- [Surrogating model components with machine learning regressions](http://nbviewer.ipython.org/github/SDXorg/PySD-Cookbook/blob/master/source/analyses/surrogating_functions/Surrogating_with_regression.ipynb) +- [Multi-Scale geographic comparison of model predictions](http://nbviewer.ipython.org/github/SDXorg/PySD-Cookbook/blob/master/source/analyses/geo/Exploring_models_across_geographic_scales.ipynb) -If you use PySD in any published work, consider citing the [PySD Introductory Paper](https://github.com/JamesPHoughton/pysd/blob/master/docs/PySD%20Intro%20Paper%20Preprint.pdf): +If you use PySD in any published work, consider citing the [PySD Introductory Paper](https://github.com/SDXorg/pysd/blob/master/docs/PySD%20Intro%20Paper%20Preprint.pdf): >Houghton, James; Siegel, Michael. "Advanced data analytics for system dynamics models using PySD." *Proceedings of the 33rd International Conference of the System Dynamics Society.* 2015. @@ -50,19 +51,17 @@ If you'd like to work with this repository directly, you'll need to use a recurs The command should be something like: ```shell -git clone --recursive https://github.com/JamesPHoughton/pysd.git +git clone --recursive https://github.com/SDXorg/pysd.git ``` ### Extensions You can use PySD in [R](https://www.r-project.org/) via the [PySD2R](https://github.com/JimDuggan/pysd2r) package, also available on [cran](https://CRAN.R-project.org/package=pysd2r). -### Contributors +### Contributing -Many people have contributed to developing this project - by -[submitting code](https://github.com/JamesPHoughton/pysd/graphs/contributors), bug reports, and advice. +PySD is currently a community-maintained project, any contribution is welcome. -Special thanks to the [sdCloud.io](http://sdcloud.io) development team, who have -made great contributions to XMILE support, and for integrating PySD into their cloud-based model simulation environment. +Many people have contributed to developing this project - by [submitting code](https://github.com/SDXorg/pysd/graphs/contributors), bug reports, and advice. Main historic changes in PySD are described in the [About PySD section](https://pysd.readthedocs.io/en/latest/about.html). The [Developer Documentation](https://pysd.readthedocs.io/en/latest/development/development_index.html) could help new developers. -Extra special thanks to [@enekomartinmartinez](https://github.com/enekomartinmartinez) for dramatically pushing forward subscript capabilities (and many other attributes). +The code for this package is available at: https://github.com/SDXorg/pysd diff --git a/docs/about.rst b/docs/about.rst index 05fa7636..6c7f0af0 100644 --- a/docs/about.rst +++ b/docs/about.rst @@ -1,6 +1,21 @@ About the Project ================= +PySD was created in 2014 by `James P Houghton `_ to translate Vensim models to Python. The original goal for translating SD models into Python was to be able to take advantage of all the tools available in Python and thus to extent what is possible using Vensim. + +Since the creation of the library, many people have contributed to the project by reporting and fixing bugs and adding new features. These contributions are listed in the `contributions section of the GitHub repository `_. + +Some of the big changes that have allowed PySD to get to its current state are the development of an XMILE to Python translator in 2017 by `Alex Prey `_ and the restructuring of the translation and model building through an Abstract Syntax by `Eneko Martin-Martinez `_ in 2022. + +Some other contributions until release 3.0.0 were: + +- `Julien Malard-Adam `_ added unicode support for the Vensim parser. +- `sdCloud.io `_ development team made great contributions to improve XMILE support and integrated PySD into their cloud-based model simulation environment. +- `Eneko Martin-Martinez `_ pushed forward the subscripts capabilities for both Vensim and XMILE and included support for several Vensim functions and improved the performance. +- `Roger Samsó `_ included a parser for the Vensim sketch and added the option to split a Vensim model per view based on the sketch information. + +The changes made since release 3.0.0 are tracked in the :doc:`whats_new` section. + Motivation: The (coming of) age of Big Data ------------------------------------------- @@ -28,5 +43,3 @@ A third category of tools imports the models created by traditional tools to per The central paradigm of PySD is that it is more efficient to bring the mature capabilities of system dynamics into an environment in use for active development in data science, than to attempt to bring each new development in inference and machine learning into the system dynamics enclave. PySD reads a model file – the product of a modeling program such as Vensim or Stella/iThink – and cross compiles it into Python, providing a simulation engine that can run these models natively in the Python environment. It is not a substitute for these tools, and cannot be used to replace a visual model construction environment. - - diff --git a/docs/conf.py b/docs/conf.py index 1fdb00fb..f4a26a0d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,8 +58,8 @@ ] extlinks = { - "issue": ("https://github.com/JamesPHoughton/pysd/issues/%s", "issue #%s"), - "pull": ("https://github.com/JamesPHoughton/pysd/pull/%s", "PR #%s"), + "issue": ("https://github.com/SDXorg/pysd/issues/%s", "issue #%s"), + "pull": ("https://github.com/SDXorg/pysd/pull/%s", "PR #%s"), } # Add any paths that contain templates here, relative to this directory. @@ -74,8 +74,8 @@ # General information about the project. project = 'PySD' -copyright = '2016, James Houghton' -author = 'James Houghton' +copyright = '2022, PySD contributors' +author = 'PySD contributors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -128,7 +128,7 @@ # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'PySD.tex', 'PySD Documentation', - 'James Houghton', 'manual'), + 'PySD contributors', 'manual'), ] # -- Options for manual page output --------------------------------------- diff --git a/docs/development/guidelines.rst b/docs/development/guidelines.rst index e1f884f8..0df3b399 100644 --- a/docs/development/guidelines.rst +++ b/docs/development/guidelines.rst @@ -25,7 +25,7 @@ of the `/tests/` directory. In order to run all the tests :py:mod:`pytest` should be used. A `Makefile` is given to run easier the tests with :py:mod:`pytest`, check -`tests/README `_ +`tests/README `_ for more information. These tests run quickly and should be executed when any changes are made to ensure diff --git a/docs/development/pathway.rst b/docs/development/pathway.rst index 0580fef2..e09187ff 100644 --- a/docs/development/pathway.rst +++ b/docs/development/pathway.rst @@ -2,7 +2,7 @@ PySD Development Pathway ======================== High priority features, bugs, and other elements of active effort are listed on the `github issue -tracker. `_ To get involved see :doc:`guidelines`. +tracker. `_ To get involved see :doc:`guidelines`. High Priority diff --git a/docs/generate_tables.py b/docs/generate_tables.py index 29fa20ca..38b6b8dd 100644 --- a/docs/generate_tables.py +++ b/docs/generate_tables.py @@ -1,6 +1,7 @@ -import pandas as pd from pathlib import Path +import pandas as pd + def generate(table, columns, output): """Generate markdown table.""" diff --git a/docs/getting_started.rst b/docs/getting_started.rst index ad00c754..bfb739bc 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -1,6 +1,14 @@ Getting Started =============== +.. note:: + A cookbook of simple recipes for advanced data analytics using PySD is available at: + http://pysd-cookbook.readthedocs.org/ + + The cookbook includes models, sample data, and code in the form of iPython notebooks that demonstrate a variety of data integration and analysis tasks. + These models can be executed on your local machine, and modified to suit your particular analysis requirements. + + Importing a model and getting started ------------------------------------- To begin, we must first load the PySD module, and use it to import a model file:: diff --git a/docs/index.rst b/docs/index.rst index 99d7d54d..0201a1a5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,20 +4,25 @@ PySD |made-with-sphinx-doc| |DOI| +|Maintained| |PyPI license| |conda package| |PyPI package| |PyPI status| |PyPI pyversions| +|Contributions| .. |made-with-sphinx-doc| image:: https://img.shields.io/badge/Made%20with-Sphinx-1f425f.svg :target: https://www.sphinx-doc.org/ +.. |Maintained| image:: https://img.shields.io/badge/Maintained-Yes-brightgreen.svg + :target: https://github.com/SDXorg/pysd/pulse + .. |docs| image:: https://readthedocs.org/projects/pysd/badge/?version=latest :target: https://pysd.readthedocs.io/en/latest/?badge=latest .. |PyPI license| image:: https://img.shields.io/pypi/l/sdqc.svg - :target: https://github.com/JamesPHoughton/pysd/blob/master/LICENSE + :target: https://github.com/SDXorg/pysd/blob/master/LICENSE .. |PyPI package| image:: https://badge.fury.io/py/pysd.svg :target: https://badge.fury.io/py/pysd @@ -34,6 +39,9 @@ PySD .. |DOI| image:: https://zenodo.org/badge/DOI/10.5281/zenodo.5654824.svg :target: https://doi.org/10.5281/zenodo.5654824 +.. |Contributions| image:: https://img.shields.io/badge/contributions-welcome-blue.svg + :target: https://pysd.readthedocs.io/en/latest/development/development_index.html + This project is a simple library for running System Dynamics models in Python, with the purpose of improving integration of Big Data and Machine Learning into the SD workflow. PySD translates :doc:`Vensim ` or @@ -63,16 +71,19 @@ The cookbook includes models, sample data, and code in the form of iPython noteb Contributing ^^^^^^^^^^^^ -The code for this package is available at: https://github.com/JamesPHoughton/pysd +|Contributions| + +PySD is currently a community-maintained project, any contribution is welcome. + +The code for this package is available at: https://github.com/SDXorg/pysd -If you find a bug, or are interested in a particular feature, see :doc:`reporting bugs <../reporting_bugs>`. +If you find any bug, or are interested in a particular feature, see :doc:`reporting bugs <../reporting_bugs>`. -If you are interested in contributing to the development of PySD, see the :doc:`developer documentation <../development/development_index>` -listed above. +If you are interested in contributing to the development of PySD, see the :doc:`developer documentation <../development/development_index>` listed above. Citing ^^^^^^ -If you use PySD in any published work, consider citing the `PySD Introductory Paper `_:: +If you use PySD in any published work, consider citing the `PySD Introductory Paper `_:: Houghton, James; Siegel, Michael. "Advanced data analytics for system dynamics models using PySD." *Proceedings of the 33rd International Conference of the System Dynamics Society.* 2015. diff --git a/docs/installation.rst b/docs/installation.rst index 60b6c4a7..6f5cf8ed 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -23,9 +23,9 @@ To install from source, clone the project with git: .. code-block:: bash - git clone https://github.com/JamesPHoughton/pysd.git + git clone https://github.com/SDXorg/pysd.git -or download the latest version from the project repository: https://github.com/JamesPHoughton/pysd +or download the latest version from the project repository: https://github.com/SDXorg/pysd In the source directory use the command: @@ -86,5 +86,5 @@ These modules can be installed using pip with a syntax similar to the above. Additional Resources -------------------- -The `PySD Cookbook `_ contains recipes that can help you get set up with PySD. +The `PySD Cookbook `_ contains recipes that can help you get set up with PySD. diff --git a/docs/reporting_bugs.rst b/docs/reporting_bugs.rst index 860a4c67..4ef4d384 100644 --- a/docs/reporting_bugs.rst +++ b/docs/reporting_bugs.rst @@ -3,7 +3,7 @@ Reporting bugs Before reporting any bug, please make sure that you are using the latest version of PySD. You can get the version number by running `python -m pysd -v` on the command line. -All bugs must be reported in the project's `issue tracker on github `_. +All bugs must be reported in the project's `issue tracker on github `_. .. note:: Not all the features and functions are implemented. If you are in trouble while translating or running a Vensim or Xmile model check the :ref:`Vensim supported functions ` or :ref:`Xmile supported functions ` and consider that when openning a new issue. diff --git a/docs/requirements.txt b/docs/requirements.txt index b97418bc..563a2af5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -4,3 +4,5 @@ sphinx==4.2.0 sphinx_rtd_theme==1.0.0 readthedocs-sphinx-search==0.1.1 jinja2==3.0.0 +mock +pandas \ No newline at end of file diff --git a/docs/whats_new.rst b/docs/whats_new.rst index dace4401..d13f6b2c 100644 --- a/docs/whats_new.rst +++ b/docs/whats_new.rst @@ -1,11 +1,11 @@ What's New ========== -v3.7.0 (to be released) +v3.7.0 (2022/09/19) ------------------- New Features ~~~~~~~~~~~~ -- Simulation results can now be stored as netCDF4 files (:issue:`355`). (`@rogersamso `_) +- Simulation results can now be stored as netCDF4 files. (`@rogersamso `_) - The CLI also accepts netCDF4 file paths after the -o argument. (`@rogersamso `_) Breaking changes @@ -23,7 +23,9 @@ Bug fixes Documentation ~~~~~~~~~~~~~ -- Adds Storing simulation results on a file section in the getting started page. (`@rogersamso `_) +- Add `Storing simulation results on a file` section in the :doc:`getting_started` page. (`@rogersamso `_) +- Include cookbook information in the :doc:`getting_started` page. (`@enekomartinmartinez `_) +- Include an introduction of main historical changes in the :doc:`about` page. (`@enekomartinmartinez `_) Performance ~~~~~~~~~~~ @@ -35,7 +37,13 @@ Internal Changes - Add netCDF4 dependency for tests. (`@rogersamso `_) - Improve warning message when replacing a stock with a parameter. (`@enekomartinmartinez `_) - Include more pytest parametrizations in some test and make them translate the models in temporary directories. (`@enekomartinmartinez `_) - +- Include lychee-action in the GHA workflow to check the links. (`@enekomartinmartinez `_) +- Update License. (`@enekomartinmartinez `_) +- Include `Maintained? Yes` and `Contributions welcome` badges. (`@enekomartinmartinez `_) +- Update links to the new repository location. (`@enekomartinmartinez `_) +- Reduce relative precision from 1e-10 to 1e-5 to compute the saving times and final time. (`@enekomartinmartinez `_) +- Add convergence tests for euler integration method. (`@enekomartinmartinez `_) +- Include build docs check in the GHA workflow to avoid warnings with sphinx. (`@enekomartinmartinez `_) v3.6.1 (2022/09/05) ------------------- @@ -60,15 +68,15 @@ Performance Internal Changes ~~~~~~~~~~~~~~~~ -- Set :py:mod:`parsimonius` requirement to 0.9.0 to avoid a breaking-change in the newest version. Pending to update PySD to run it with :py:mod:`parsimonious` 0.10.0. +- Set :py:mod:`parsimonius` requirement to 0.9.0 to avoid a breaking-change in the newest version. Pending to update PySD to run it with :py:mod:`parsimonious` 0.10.0. (`@enekomartinmartinez `_) v3.6.0 (2022/08/31) ------------------- New Features ~~~~~~~~~~~~ -- Include warning messages when a variable is defined in more than one view, when a control variable appears in a view or when a variable doesn't appear in any view as a `workbench variable` (:issue:`357`). -- Force variables in a module to be saved alphabetically for being able to compare differences between versions (only for the models that are split by views). +- Include warning messages when a variable is defined in more than one view, when a control variable appears in a view or when a variable doesn't appear in any view as a `workbench variable` (:issue:`357`). (`@enekomartinmartinez `_) +- Force variables in a module to be saved alphabetically for being able to compare differences between versions (only for the models that are split by views). (`@enekomartinmartinez `_) Breaking changes ~~~~~~~~~~~~~~~~ @@ -78,7 +86,7 @@ Deprecations Bug fixes ~~~~~~~~~ -- Classify control variables in the main file always (:issue:`357`). +- Classify control variables in the main file always (:issue:`357`). (`@enekomartinmartinez `_) Documentation ~~~~~~~~~~~~~ @@ -88,7 +96,7 @@ Performance Internal Changes ~~~~~~~~~~~~~~~~ -- Include :py:class:`pysd.translators.structures.abstract_model.AbstractControlElement` child of :py:class:`pysd.translators.structures.abstract_model.AbstractElement` to differentiate the control variables. +- Include :py:class:`pysd.translators.structures.abstract_model.AbstractControlElement` child of :py:class:`pysd.translators.structures.abstract_model.AbstractElement` to differentiate the control variables. (`@enekomartinmartinez `_) v3.5.2 (2022/08/15) @@ -105,7 +113,7 @@ Deprecations Bug fixes ~~~~~~~~~ -- Make sketch's `font_size` optional. +- Make sketch's `font_size` optional. (`@enekomartinmartinez `_) Documentation ~~~~~~~~~~~~~ @@ -131,16 +139,16 @@ Deprecations Bug fixes ~~~~~~~~~ -- Fix bug generated when :EXCEPT: keyword is used with subscript subranges (:issue:`352`). -- Fix bug of precision error for :py:func:`pysd.py_backend.allocation.allocate_by_priority` (:issue:`353`). -- Fix bug of constant cache assignment. +- Fix bug generated when :EXCEPT: keyword is used with subscript subranges (:issue:`352`). (`@enekomartinmartinez `_) +- Fix bug of precision error for :py:func:`pysd.py_backend.allocation.allocate_by_priority` (:issue:`353`). (`@enekomartinmartinez `_) +- Fix bug of constant cache assignment. (`@enekomartinmartinez `_) Documentation ~~~~~~~~~~~~~ Performance ~~~~~~~~~~~ -- Improve the performance of reading :py:class:`pysd.py_backend.external.External` data with cellrange names by loading the data in memory with :py:mod:`pandas`. As recommended by :py:mod:`openpyxl` developers, this is a possible way of improving performance to avoid parsing all rows up each time for getting the data (`issue 1867 in openpyxl `_). +- Improve the performance of reading :py:class:`pysd.py_backend.external.External` data with cellrange names by loading the data in memory with :py:mod:`pandas`. As recommended by :py:mod:`openpyxl` developers, this is a possible way of improving performance to avoid parsing all rows up each time for getting the data (`issue 1867 in openpyxl `_). (`@enekomartinmartinez `_) Internal Changes ~~~~~~~~~~~~~~~~ @@ -150,7 +158,7 @@ v3.5.0 (2022/07/25) New Features ~~~~~~~~~~~~ -- Add support for subscripted arguments in :py:func:`pysd.py_backend.functions.ramp` and :py:func:`pysd.py_backend.functions.step` (:issue:`344`). +- Add support for subscripted arguments in :py:func:`pysd.py_backend.functions.ramp` and :py:func:`pysd.py_backend.functions.step` (:issue:`344`). (`@enekomartinmartinez `_) Breaking changes ~~~~~~~~~~~~~~~~ @@ -160,21 +168,21 @@ Deprecations Bug fixes ~~~~~~~~~ -- Fix bug related to the order of elements in 1D GET expressions (:issue:`343`). -- Fix bug in request 0 values in allocate by priority (:issue:`345`). -- Fix a numerical error in starting time of step and ramp. +- Fix bug related to the order of elements in 1D GET expressions (:issue:`343`). (`@enekomartinmartinez `_) +- Fix bug in request 0 values in allocate by priority (:issue:`345`). (`@enekomartinmartinez `_) +- Fix a numerical error in starting time of step and ramp. (`@enekomartinmartinez `_) Documentation ~~~~~~~~~~~~~ -- Include new PySD logo. +- Include new PySD logo. (`@enekomartinmartinez `_) Performance ~~~~~~~~~~~ Internal Changes ~~~~~~~~~~~~~~~~ -- Ignore 'distutils Version classes are deprecated. Use packaging.version instead' error in tests as it is an internal error of `xarray`. -- Add a warning message when a subscript range is duplicated in a variable reference. +- Ignore 'distutils Version classes are deprecated. Use packaging.version instead' error in tests as it is an internal error of `xarray`. (`@enekomartinmartinez `_) +- Add a warning message when a subscript range is duplicated in a variable reference. (`@enekomartinmartinez `_) v3.4.0 (2022/06/29) @@ -182,7 +190,7 @@ v3.4.0 (2022/06/29) New Features ~~~~~~~~~~~~ -- Add support for Vensim's `ALLOCATE AVAILABLE `_ (:py:func:`pysd.py_backend.allocation.allocate_available`) function (:issue:`339`). Integer allocation cases have not been implemented neither the fixed quantity and constant elasticity curve priority functions. +- Add support for Vensim's `ALLOCATE AVAILABLE `_ (:py:func:`pysd.py_backend.allocation.allocate_available`) function (:issue:`339`). Integer allocation cases have not been implemented neither the fixed quantity and constant elasticity curve priority functions. (`@enekomartinmartinez `_) Breaking changes ~~~~~~~~~~~~~~~~ @@ -195,14 +203,14 @@ Bug fixes Documentation ~~~~~~~~~~~~~ -- Improve the documentation of the :py:mod:`pysd.py_backend.allocation` module. +- Improve the documentation of the :py:mod:`pysd.py_backend.allocation` module. (`@enekomartinmartinez `_) Performance ~~~~~~~~~~~ Internal Changes ~~~~~~~~~~~~~~~~ -- Add a class to manage priority profiles so it can be also used by the `many-to-many allocation `_. +- Add a class to manage priority profiles so it can be also used by the `many-to-many allocation `_. (`@enekomartinmartinez `_) v3.3.0 (2022/06/22) @@ -210,7 +218,7 @@ v3.3.0 (2022/06/22) New Features ~~~~~~~~~~~~ -- Add support for Vensim's `ALLOCATE BY PRIORITY `_ (:py:func:`pysd.py_backend.allocation.allocate_by_priority`) function (:issue:`263`). +- Add support for Vensim's `ALLOCATE BY PRIORITY `_ (:py:func:`pysd.py_backend.allocation.allocate_by_priority`) function (:issue:`263`). (`@enekomartinmartinez `_) Breaking changes ~~~~~~~~~~~~~~~~ @@ -220,7 +228,7 @@ Deprecations Bug fixes ~~~~~~~~~ -- Fix bug of using subranges to define a bigger range (:issue:`335`). +- Fix bug of using subranges to define a bigger range (:issue:`335`). (`@enekomartinmartinez `_) Documentation ~~~~~~~~~~~~~ @@ -230,15 +238,15 @@ Performance Internal Changes ~~~~~~~~~~~~~~~~ -- Improve error messages for :class:`pysd.py_backend.External` objects. +- Improve error messages for :class:`pysd.py_backend.External` objects. (`@enekomartinmartinez `_) v3.2.0 (2022/06/10) ------------------- New Features ~~~~~~~~~~~~ -- Add support for Vensim's `GET TIME VALUE `_ (:py:func:`pysd.py_backend.functions.get_time_value`) function (:issue:`332`). Not all cases have been implemented. -- Add support for Vensim's `VECTOR SELECT `_ (:py:func:`pysd.py_backend.functions.vector_select`) function (:issue:`266`). +- Add support for Vensim's `GET TIME VALUE `_ (:py:func:`pysd.py_backend.functions.get_time_value`) function (:issue:`332`). Not all cases have been implemented. (`@enekomartinmartinez `_) +- Add support for Vensim's `VECTOR SELECT `_ (:py:func:`pysd.py_backend.functions.vector_select`) function (:issue:`266`). (`@enekomartinmartinez `_) Breaking changes ~~~~~~~~~~~~~~~~ @@ -265,9 +273,9 @@ v3.1.0 (2022/06/02) New Features ~~~~~~~~~~~~ -- Add support for Vensim's `VECTOR SORT ORDER `_ (:py:func:`pysd.py_backend.functions.vector_sort_order`) function (:issue:`326`). -- Add support for Vensim's `VECTOR RANK `_ (:py:func:`pysd.py_backend.functions.vector_rank`) function (:issue:`326`). -- Add support for Vensim's `VECTOR REORDER `_ (:py:func:`pysd.py_backend.functions.vector_reorder`) function (:issue:`326`). +- Add support for Vensim's `VECTOR SORT ORDER `_ (:py:func:`pysd.py_backend.functions.vector_sort_order`) function (:issue:`326`). (`@enekomartinmartinez `_) +- Add support for Vensim's `VECTOR RANK `_ (:py:func:`pysd.py_backend.functions.vector_rank`) function (:issue:`326`). (`@enekomartinmartinez `_) +- Add support for Vensim's `VECTOR REORDER `_ (:py:func:`pysd.py_backend.functions.vector_reorder`) function (:issue:`326`). (`@enekomartinmartinez `_) Breaking changes ~~~~~~~~~~~~~~~~ @@ -280,7 +288,7 @@ Bug fixes Documentation ~~~~~~~~~~~~~ -- Add the section :doc:`/development/adding_functions` with examples for developers. +- Add the section :doc:`/development/adding_functions` with examples for developers. (`@enekomartinmartinez `_) Performance ~~~~~~~~~~~ @@ -306,7 +314,7 @@ Deprecations Bug fixes ~~~~~~~~~ -- Simplify subscripts dictionaries for :py:class:`pysd.py_backend.data.TabData` objects. +- Simplify subscripts dictionaries for :py:class:`pysd.py_backend.data.TabData` objects. (`@enekomartinmartinez `_) Documentation ~~~~~~~~~~~~~ @@ -318,12 +326,12 @@ Performance Internal Changes ~~~~~~~~~~~~~~~~ -- Add Python 3.10 to CI pipeline and include it in the supported versions list. -- Correct LICENSE file extension in the `setup.py`. -- Move from `importlib`'s :py:func:`load_module` to :py:func:`exec_module`. -- Remove warnings related to :py:data:`set` usage. -- Move all the missing test to :py:mod:`pytest`. -- Remove warning messages from test and make test fail if there is any warning. +- Add Python 3.10 to CI pipeline and include it in the supported versions list. (`@enekomartinmartinez `_) +- Correct LICENSE file extension in the `setup.py`. (`@enekomartinmartinez `_) +- Move from `importlib`'s :py:func:`load_module` to :py:func:`exec_module`. (`@enekomartinmartinez `_) +- Remove warnings related to :py:data:`set` usage. (`@enekomartinmartinez `_) +- Move all the missing test to :py:mod:`pytest`. (`@enekomartinmartinez `_) +- Remove warning messages from test and make test fail if there is any warning. (`@enekomartinmartinez `_) v3.0.0 (2022/05/23) @@ -332,9 +340,9 @@ v3.0.0 (2022/05/23) New Features ~~~~~~~~~~~~ -- The new :doc:`Abstract Model Representation ` translation and building workflow will allow to add new output languages in the future. -- Added new properties to the :py:class:`pysd.py_backend.model.Macro` to make more accessible some information: :py:attr:`.namespace`, :py:attr:`.subscripts`, :py:attr:`.dependencies`, :py:attr:`.modules`, :py:attr:`.doc`. -- Cleaner Python models: +- The new :doc:`Abstract Model Representation ` translation and building workflow will allow to add new output languages in the future. (`@enekomartinmartinez `_) +- Added new properties to the :py:class:`pysd.py_backend.model.Macro` to make more accessible some information: :py:attr:`.namespace`, :py:attr:`.subscripts`, :py:attr:`.dependencies`, :py:attr:`.modules`, :py:attr:`.doc`. (`@enekomartinmartinez `_) +- Cleaner Python models: (`@enekomartinmartinez `_) - :py:data:`_namespace` and :py:data:`_dependencies` dictionaries have been removed from the file. - Variables original names, dependencies metadata now are given through :py:meth:`pysd.py_backend.components.Component.add` decorator, instead of having them in the docstring. - Merging of variable equations is now done using the coordinates to a pre-allocated array, instead of using the `magic` function :py:data:`pysd.py_backend.utils.xrmerge()`. @@ -343,48 +351,48 @@ New Features Breaking changes ~~~~~~~~~~~~~~~~ -- Set the argument :py:data:`flatten_output` from :py:meth:`.run` to :py:data:`True` by default. Previously it was set to :py:data:`False` by default. -- Move the docstring of the model to a property, :py:attr:`.doc`. Thus, it is not callable anymore. -- Allow the function :py:func:`pysd.py_backend.functions.pulse` to also perform the operations performed by :py:data:`pysd.py_backend.functions.pulse_train()` and :py:data:`pysd.py_backend.functions.pulse_magnitude()`. -- Change first argument of :py:func:`pysd.py_backend.functions.active_initial`, now it is the `stage of the model` and not the `time`. -- Simplify the function :py:data:`pysd.py_backend.utils.rearrange()` orienting it to perform simple rearrange cases for user interaction. -- Move :py:data:`pysd.py_backend.statefuls.Model` and :py:data:`pysd.py_backend.statefuls.Macro` to :py:class:`pysd.py_backend.model.Model` and :py:class:`pysd.py_backend.model.Macro`, respectively. -- Manage all kinds of lookups with the :py:class:`pysd.py_backend.lookups.Lookups` class. -- Include a second optional argument to lookups functions to set the final coordinates when a subscripted variable is passed as an argument. +- Set the argument :py:data:`flatten_output` from :py:meth:`.run` to :py:data:`True` by default. Previously it was set to :py:data:`False` by default. (`@enekomartinmartinez `_) +- Move the docstring of the model to a property, :py:attr:`.doc`. Thus, it is not callable anymore. (`@enekomartinmartinez `_) +- Allow the function :py:func:`pysd.py_backend.functions.pulse` to also perform the operations performed by :py:data:`pysd.py_backend.functions.pulse_train()` and :py:data:`pysd.py_backend.functions.pulse_magnitude()`. (`@enekomartinmartinez `_) +- Change first argument of :py:func:`pysd.py_backend.functions.active_initial`, now it is the `stage of the model` and not the `time`. (`@enekomartinmartinez `_) +- Simplify the function :py:data:`pysd.py_backend.utils.rearrange()` orienting it to perform simple rearrange cases for user interaction. (`@enekomartinmartinez `_) +- Move :py:data:`pysd.py_backend.statefuls.Model` and :py:data:`pysd.py_backend.statefuls.Macro` to :py:class:`pysd.py_backend.model.Model` and :py:class:`pysd.py_backend.model.Macro`, respectively. (`@enekomartinmartinez `_) +- Manage all kinds of lookups with the :py:class:`pysd.py_backend.lookups.Lookups` class. (`@enekomartinmartinez `_) +- Include a second optional argument to lookups functions to set the final coordinates when a subscripted variable is passed as an argument. (`@enekomartinmartinez `_) Deprecations ~~~~~~~~~~~~ -- Remove :py:data:`pysd.py_backend.utils.xrmerge()`, :py:data:`pysd.py_backend.functions.pulse_train()`, :py:data:`pysd.py_backend.functions.pulse_magnitude()`, :py:data:`pysd.py_backend.functions.lookup()`, :py:data:`pysd.py_backend.functions.lookup_discrete()`, :py:data:`pysd.py_backend.functions.lookup_extrapolation()`, :py:data:`pysd.py_backend.functions.logical_and()`, :py:data:`pysd.py_backend.functions.logical_or()`, :py:data:`pysd.py_backend.functions.bounded_normal()`, :py:data:`pysd.py_backend.functions.log()`. -- Remove old translation and building files (:py:data:`pysd.translation`). +- Remove :py:data:`pysd.py_backend.utils.xrmerge()`, :py:data:`pysd.py_backend.functions.pulse_train()`, :py:data:`pysd.py_backend.functions.pulse_magnitude()`, :py:data:`pysd.py_backend.functions.lookup()`, :py:data:`pysd.py_backend.functions.lookup_discrete()`, :py:data:`pysd.py_backend.functions.lookup_extrapolation()`, :py:data:`pysd.py_backend.functions.logical_and()`, :py:data:`pysd.py_backend.functions.logical_or()`, :py:data:`pysd.py_backend.functions.bounded_normal()`, :py:data:`pysd.py_backend.functions.log()`. (`@enekomartinmartinez `_) +- Remove old translation and building files (:py:data:`pysd.translation`). (`@enekomartinmartinez `_) Bug fixes ~~~~~~~~~ -- Generate the documentation of the model when loading it to avoid lossing information when replacing a variable value (:issue:`310`, :pull:`312`). -- Make random functions return arrays of the same shape as the variable, to avoid repeating values over a dimension (:issue:`309`, :pull:`312`). -- Fix bug when Vensim's :MACRO: definition is not at the top of the model file (:issue:`306`, :pull:`312`). -- Make builder identify the subscripts using a main range and subrange to allow using subscripts as numeric values as Vensim does (:issue:`296`, :issue:`301`, :pull:`312`). -- Fix bug of missmatching of functions and lookups names (:issue:`116`, :pull:`312`). -- Parse Xmile models case insensitively and ignoring the new lines characters (:issue:`203`, :issue:`253`, :pull:`312`). -- Add support for Vensim's `\:EXCEPT\: keyword `_ (:issue:`168`, :issue:`253`, :pull:`312`). -- Add spport for Xmile's FORCST and SAFEDIV functions (:issue:`154`, :pull:`312`). -- Add subscripts support for Xmile (:issue:`289`, :pull:`312`). -- Fix numeric error bug when using :py:data:`return_timestamps` and time step with non-integer values. +- Generate the documentation of the model when loading it to avoid lossing information when replacing a variable value (:issue:`310`, :pull:`312`). (`@enekomartinmartinez `_) +- Make random functions return arrays of the same shape as the variable, to avoid repeating values over a dimension (:issue:`309`, :pull:`312`). (`@enekomartinmartinez `_) +- Fix bug when Vensim's :MACRO: definition is not at the top of the model file (:issue:`306`, :pull:`312`). (`@enekomartinmartinez `_) +- Make builder identify the subscripts using a main range and subrange to allow using subscripts as numeric values as Vensim does (:issue:`296`, :issue:`301`, :pull:`312`). (`@enekomartinmartinez `_) +- Fix bug of missmatching of functions and lookups names (:issue:`116`, :pull:`312`). (`@enekomartinmartinez `_) +- Parse Xmile models case insensitively and ignoring the new lines characters (:issue:`203`, :issue:`253`, :pull:`312`). (`@enekomartinmartinez `_) +- Add support for Vensim's `\:EXCEPT\: keyword `_ (:issue:`168`, :issue:`253`, :pull:`312`). (`@enekomartinmartinez `_) +- Add spport for Xmile's FORCST and SAFEDIV functions (:issue:`154`, :pull:`312`). (`@enekomartinmartinez `_) +- Add subscripts support for Xmile (:issue:`289`, :pull:`312`). (`@enekomartinmartinez `_) +- Fix numeric error bug when using :py:data:`return_timestamps` and time step with non-integer values. (`@enekomartinmartinez `_) Documentation ~~~~~~~~~~~~~ -- Review the whole documentation, refract it, and describe the new features. +- Review the whole documentation, refract it, and describe the new features. (`@enekomartinmartinez `_) Performance ~~~~~~~~~~~ -- The variables defined in several equations are now assigned to a pre-allocated array instead of using :py:data:`pysd.py_backend.utils.xrmerge()`. -- The arranging and subseting of arrays is now done inplace instead of using the magic function :py:data:`pysd.py_backend.utils.rearrange()`. -- The grammars for Parsimonious are only compiled once per translation. +- The variables defined in several equations are now assigned to a pre-allocated array instead of using :py:data:`pysd.py_backend.utils.xrmerge()`. (`@enekomartinmartinez `_) +- The arranging and subseting of arrays is now done inplace instead of using the magic function :py:data:`pysd.py_backend.utils.rearrange()`. (`@enekomartinmartinez `_) +- The grammars for Parsimonious are only compiled once per translation. (`@enekomartinmartinez `_) Internal Changes ~~~~~~~~~~~~~~~~ -- The translation and the building of models has been totally modified to use the :doc:`Abstract Model Representation `. +- The translation and the building of models has been totally modified to use the :doc:`Abstract Model Representation `. (`@enekomartinmartinez `_) diff --git a/pysd/py_backend/components.py b/pysd/py_backend/components.py index 20fac9f3..140ac9b2 100644 --- a/pysd/py_backend/components.py +++ b/pysd/py_backend/components.py @@ -123,7 +123,7 @@ def _set_component(self, name, value): class Time(object): - rprec = 1e-10 # relative precission for final time and saving time + rprec = 1e-5 # relative precision for final time and saving time def __init__(self): self._time = None @@ -183,7 +183,7 @@ def in_return(self): prec = self.time_step() * self.rprec if self.return_timestamps is not None: - # this allows managing float precission error + # this allows managing float precision error if self.next_return is None: return False if np.isclose(self._time, self.next_return, prec): @@ -207,7 +207,7 @@ def in_return(self): return time_delay % save_per < prec or -time_delay % save_per < prec def round(self): - """ Return rounded time to outputs to avoid float precission error""" + """ Return rounded time to outputs to avoid float precision error""" return np.round( self._time, -int(np.log10(self.time_step()*self.rprec))) diff --git a/pysd/pysd.py b/pysd/pysd.py index 09bcdd64..fc032d6b 100644 --- a/pysd/pysd.py +++ b/pysd/pysd.py @@ -20,7 +20,7 @@ + "." + "\nPlease update your Python version or use the last " + " supported version:\n\t" - + "https://github.com/JamesPHoughton/pysd/releases/tag/LastPy2" + + "https://github.com/SDXorg/pysd/releases/tag/LastPy2" ) diff --git a/setup.py b/setup.py index be4253d6..7f9371e4 100755 --- a/setup.py +++ b/setup.py @@ -7,10 +7,9 @@ name='pysd', version=__version__, python_requires='>=3.7', - author='James Houghton', - author_email='james.p.houghton@gmail.com', + author='PySD contributors', packages=find_packages(exclude=['docs', 'tests', 'dist', 'build']), - url='https://github.com/JamesPHoughton/pysd', + url='https://github.com/SDXorg/pysd', license='LICENSE', description='System Dynamics Modeling in Python', long_description=open('README.md').read(), diff --git a/tests/pytest.ini b/tests/pytest.ini index b3e76fa3..ac62136e 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -3,5 +3,4 @@ python_files = pytest_*/**/*.py pytest_*/*.py filterwarnings = error ignore:Creating an ndarray from ragged nested sequences - ignore:distutils Version classes are deprecated. Use packaging.version instead ignore:`np.bool` is a deprecated alias for the builtin `bool`. To silence this warning, use `bool` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.bool_` here. diff --git a/tests/pytest_integration/pytest_integration_euler.py b/tests/pytest_integration/pytest_integration_euler.py new file mode 100644 index 00000000..33ede4c4 --- /dev/null +++ b/tests/pytest_integration/pytest_integration_euler.py @@ -0,0 +1,211 @@ +from pathlib import Path + +import pytest +import numpy as np +import pandas as pd + + +def harmonic_position(t, x0, k, m): + """ + Position for the simple harmonic oscillator + 'test-models/samples/simple_harmonic_oscillator + /simple_harmonic_oscillator.mdl' + """ + return x0*np.cos(np.sqrt(k/m)*t) + + +def harmonic_speed(t, x0, k, m): + """ + Speed for the simple harmonic oscillator + 'test-models/samples/simple_harmonic_oscillator + /simple_harmonic_oscillator.mdl' + """ + return - x0*np.sqrt(k/m)*np.sin(np.sqrt(k/m)*t) + + +@pytest.mark.parametrize( + "model_path,f,f2,stocks,arguments,integration_frame,ts_log,ts_rmse", + [ + # model_path: model path: pathlib.Path object + # f: stocks analytical solutions: tuple of funcs + # f2: stocks analytical solutions second derivative: tuple of funcs + # stocks: stock names in the model: tuple of strings + # arguments: arguments of fs and f2s in the model: tuple of strings + # integration_frame: minimum and maximum time to find solutions: tuple + # ts_log: logarithmic range of time steps in base 10: tuple + # ts_rmse: sorted time step for RMSE test: iterable + ( + Path("test-models/samples/teacup/teacup.mdl"), + (lambda t, T0, TR, ct: TR + (T0-TR)*np.exp(-t/ct),), + (lambda t, T0, TR, ct: (T0-TR)*np.exp(-t/ct)/ct**2,), + ("Teacup Temperature",), + ("Teacup Temperature", "Room Temperature", "Characteristic Time"), + (0, 40), + (1, -5), + [10, 5, 1, 0.5, 0.1, 0.05, 0.01] + ), + ( + Path("test-models/samples/simple_harmonic_oscillator/" + "simple_harmonic_oscillator.mdl"), + ( + harmonic_position, + harmonic_speed + ), + ( + lambda t, x0, k, m: -k/m*harmonic_position(t, x0, k, m), + lambda t, x0, k, m: -k/m*harmonic_speed(t, x0, k, m) + ), + ("position", "speed"), + ("initial position", "elastic constant", "mass"), + (0, 40), + (-1, -5), + [10, 5, 1, 0.5, 0.1, 0.05, 0.01] + ) + ], + ids=["teacup", "harmonic"] +) +class TestEulerConvergence: + """ + Tests for Euler integration method convergence. + """ + # Number of points to compute the tests + n_points_lte = 30 + + def test_local_truncation_error(self, model, f, f2, stocks, arguments, + integration_frame, ts_log, ts_rmse): + """ + Test the local truncation error (LTE). + LTE = y_1 - y(t_0+h) = 0.5*h**2*y''(x) for x in [t_0, t_0+h] + + where y_1 = y(t_0) + h*f(t_0, y(t_0)) and + + Generates n_points_lte in the given integration frame and test the + convergence with logarithmically uniform split time_steps. + + Parameter + --------- + model: pysd.py_backend.model.Model + The model to integrate. + f: tuple of functions + The functions of the analytical solution of each stock. + f2: tuple of functions + The second derivative of the functions of the analytical + solution of each stock. + stocks: tuple of strings + The name of the stocks. + arguments: tuple of strings + The neccessary argument names to evaluate f's and f2's. + Note that all the functions must take the same arguments + and in the same order. + integration_frame: tuple + Initial time of the model (usually 0) and maximum time to + generate a value for test the LTE. + ts_log: tuple + log in base 10 of the inteval of time step to generate. I.e., + the first point will be evaluated with time_step = 10**ts_log[0] + and the last one with 10**ts_log[1]. + ts_rmse: iterable + Not used. + + """ + # Generate starting points to compute LTE + t0s = np.random.uniform(*integration_frame, self.n_points_lte) + # Generate time steps + hs = 10**np.linspace(*ts_log, self.n_points_lte) + # Get model values before making any change + model_values = [model[var] for var in arguments] + + for t0, h in zip(t0s, hs): + # Reload model + model.reload() + # Get start value(s) + x0s = [ + func(t0, *model_values) + for func in f + ] + # Get expected value(s) + x_expect = np.array([ + func(t0 + h, *model_values) + for func in f + ]) + # Get error bound (error = 0.5h²*f''(x) for x in [t0, t0+h]) + # The 0.5 factor is removed to avoid problems with local maximums + # We assume error < 2h²*max(f''(x)) for x in [t0, t0+h] + error = 2*h**2*np.array([ + max(func(np.linspace(t0, t0+h, 1000), *model_values)) + for func in f2 + ]) + # Run the model from (t0, x0s) to t0+h + ic = t0, {stock: x0 for stock, x0 in zip(stocks, x0s)} + x_euler = model.run( + initial_condition=ic, + time_step=h, + return_columns=stocks, + return_timestamps=t0+h + ).values[0] + + # Expected error + assert np.all(np.abs(x_expect - x_euler) <= np.abs(error)),\ + f"The LTE is bigger than the expected one ({ic}) h={h},"\ + f"\n{np.abs(x_expect - x_euler)} !<= {np.abs(error)}, " + + def test_root_mean_square_error(self, model, f, f2, stocks, arguments, + integration_frame, ts_log, ts_rmse): + """ + Test the root-mean-square error (RMSE). + RMSE = SQRT(MEAN((y_i-y(t_0+h*i))^2)) + + Integrates the given model with different time steps and checks + that the RMSE decreases when the time step decreases. + + Parameter + --------- + model: pysd.py_backend.model.Model + The model to integrate. + f: tuple of functions + The functions of the analytical solution of each stock. + f2: tuple of functions + Not used. + stocks: tuple of strings + The name of the stocks. + arguments: tuple of strings + The neccessary argument names to evaluate f's and f2's. + Note that all the functions must take the same arguments + and in the same order. + integration_frame: tuple + Not used. + ts_log: tuple + Not used. + ts_rmse: iterable + Time step to compute the root mean square error over the + whole integration. It shopuld be sorted from biggest to + smallest. + + """ + # Get model values before making any change + model_values = [model[var] for var in arguments] + + rmse = [] + for h in ts_rmse: + # Reload model + model.reload() + # Run the model from (t0, x0s) to t0+h + x_euler = model.run( + time_step=h, + saveper=h, + return_columns=stocks + ) + # Expected values + expected_values = pd.DataFrame( + index=x_euler.index, + data={ + stock: func(x_euler.index, *model_values) + for stock, func in zip(stocks, f) + } + ) + # Compute the RMSE for each stock + rmse.append(np.sqrt(((x_euler-expected_values)**2).mean())) + + # Assert that the RMSE decreases for all stocks while + # decreasing the time step + assert np.all(np.diff(rmse, axis=0) < 0) diff --git a/tests/pytest_pysd/pytest_pysd.py b/tests/pytest_pysd/pytest_pysd.py index a71e227b..f87fecaa 100644 --- a/tests/pytest_pysd/pytest_pysd.py +++ b/tests/pytest_pysd/pytest_pysd.py @@ -109,7 +109,7 @@ def test_run_progress_dynamic(self, model): @pytest.mark.parametrize("model_path", [test_model]) def test_run_return_timestamps(self, model): - """Addresses https://github.com/JamesPHoughton/pysd/issues/17""" + """Addresses https://github.com/SDXorg/pysd/issues/17""" timestamps = np.random.randint(1, 5, 5).cumsum() stocks = model.run(return_timestamps=timestamps) assert (stocks.index.values == timestamps).all() @@ -183,7 +183,7 @@ def test_return_timestamps_with_range(self, model): @pytest.mark.parametrize("model_path", [test_model]) def test_run_return_columns_original_names(self, model): """ - Addresses https://github.com/JamesPHoughton/pysd/issues/26 + Addresses https://github.com/SDXorg/pysd/issues/26 - Also checks that columns are returned in the correct order """ return_columns = ["Room Temperature", "Teacup Temperature"] @@ -201,7 +201,7 @@ def test_run_return_columns_step(self, model): @pytest.mark.parametrize("model_path", [test_model]) def test_run_reload(self, model): - """Addresses https://github.com/JamesPHoughton/pysd/issues/99""" + """Addresses https://github.com/SDXorg/pysd/issues/99""" result0 = model.run() result1 = model.run(params={"Room Temperature": 1000}) result2 = model.run() @@ -213,7 +213,7 @@ def test_run_reload(self, model): @pytest.mark.parametrize("model_path", [test_model]) def test_run_return_columns_pysafe_names(self, model): - """Addresses https://github.com/JamesPHoughton/pysd/issues/26""" + """Addresses https://github.com/SDXorg/pysd/issues/26""" return_columns = ["room_temperature", "teacup_temperature"] result = model.run(return_columns=return_columns) assert set(result.columns) == set(return_columns) @@ -237,7 +237,7 @@ def test_initial_conditions_tuple_pysafe_names(self, model): @pytest.mark.parametrize("model_path", [test_model]) def test_initial_conditions_tuple_original_names(self, model): - """ Responds to https://github.com/JamesPHoughton/pysd/issues/77""" + """ Responds to https://github.com/SDXorg/pysd/issues/77""" stocks = model.run( initial_condition=(3000, {"Teacup Temperature": 33}), return_timestamps=list(range(3000, 3010)), @@ -270,8 +270,7 @@ def test_initial_conditions_subscripted_value_with_numpy_error(self, @pytest.mark.parametrize("model_path", [test_model]) def test_set_constant_parameter(self, model): - """ In response to: - re: https://github.com/JamesPHoughton/pysd/issues/5""" + """Responds to https://github.com/SDXorg/pysd/issues/5""" model.set_components({"room_temperature": 20}) assert model.components.room_temperature() == 20 @@ -330,7 +329,7 @@ def test_set_component_with_real_name(self, model): @pytest.mark.parametrize("model_path", [test_model]) def test_set_components_warnings(self, model): - """Addresses https://github.com/JamesPHoughton/pysd/issues/80""" + """Addresses https://github.com/SDXorg/pysd/issues/80""" warn_message = r"Replacing the equation of stock "\ r"'Teacup Temperature' with params\.\.\." with pytest.warns(UserWarning, match=warn_message): @@ -1258,7 +1257,7 @@ def test_default_returns_with_construction_functions(self, _root): [Path("test-models/tests/lookups/test_lookups.mdl")]) def test_default_returns_with_lookups(self, model): """ - Addresses https://github.com/JamesPHoughton/pysd/issues/114 + Addresses https://github.com/SDXorg/pysd/issues/114 The default settings should skip model elements with no particular return value """ @@ -1268,7 +1267,7 @@ def test_default_returns_with_lookups(self, model): @pytest.mark.parametrize("model_path", [test_model]) def test_files(self, model, model_path, tmp_path): - """Addresses https://github.com/JamesPHoughton/pysd/issues/86""" + """Addresses https://github.com/SDXorg/pysd/issues/86""" # Path from where the model is translated path = tmp_path / model_path.parent.name / model_path.name @@ -1291,7 +1290,7 @@ def test_multiple_load(self, _root): attributes This test responds to issue: - https://github.com/JamesPHoughton/pysd/issues/23 + https://github.com/SDXorg/pysd/issues/23 """ @@ -1311,7 +1310,7 @@ def test_no_crosstalk(self, _root): Need to check that if we instantiate two copies of the same model, changes to one copy do not influence the other copy. - Checks for issue: https://github.com/JamesPHoughton/pysd/issues/108 + Checks for issue: https://github.com/SDXorg/pysd/issues/108 that time is not shared between the two models """ diff --git a/tests/test-models b/tests/test-models index 03e2369c..a19097ce 160000 --- a/tests/test-models +++ b/tests/test-models @@ -1 +1 @@ -Subproject commit 03e2369cc68c51cf8acb6821601cc1c26c79d47d +Subproject commit a19097cee55a631652feeb5ecf74adcadea549e6