From 08156b541e1581a135ae749e145ab7d6d51ade42 Mon Sep 17 00:00:00 2001 From: Eneko Martin-Martinez <61285767+enekomartinmartinez@users.noreply.github.com> Date: Thu, 2 Jun 2022 15:32:14 +0200 Subject: [PATCH] Add support for vector ordering functions and add examples of development to docs (#327) * Add support for vector ordering functions * Add examples of development to docs --- .github/pull_request_template.md | 25 ++++ docs/development/adding_functions.rst | 122 ++++++++++++++++++ docs/development/development_index.rst | 2 + docs/structure/vensim_translation.rst | 1 + docs/tables/functions.tab | 5 +- docs/whats_new.rst | 31 +++++ pysd/_version.py | 2 +- pysd/builders/python/python_functions.py | 9 ++ pysd/builders/python/python_model_builder.py | 4 +- pysd/py_backend/functions.py | 108 +++++++++++++++- .../pytest_integration_vensim_pathway.py | 6 +- tests/test-models | 2 +- 12 files changed, 310 insertions(+), 7 deletions(-) create mode 100644 .github/pull_request_template.md create mode 100644 docs/development/adding_functions.rst diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..cfd89e9a --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,25 @@ +## Description + +(add the description of your changes here) + +## Related issues + +(add any related issues here) + +## Type of change + +(please delete options that are not relevant) + +- Bug fix (non-breaking change which fixes an issue) +- New feature (non-breaking change which adds functionality) +- Breaking change (fix or feature that would cause existing functionality to not work as expected) +- Other (specify) + +## PR verification (to be filled by reviewers) + +- [ ] The code follows the [PEP 8 style](https://peps.python.org/pep-0008/) +- [ ] The new code has been tested properly for all the possible cases +- [ ] The overall coverage has not dropped and other features have not been broken. +- [ ] If new features have been added, they have been properly documented +- [ ] *docs/whats_new.rst* has been updated +- [ ] PySD version has been updated diff --git a/docs/development/adding_functions.rst b/docs/development/adding_functions.rst new file mode 100644 index 00000000..c947aedd --- /dev/null +++ b/docs/development/adding_functions.rst @@ -0,0 +1,122 @@ +Adding new functions +==================== +In this section you may found some helpful examples for adding a new function to the PySD Python builder. Before starting adding any new feature or fuction, please, make sure that no one is working on it. Search if any open issue exists with the feature you want to work on or open a new one if it does not exist. Then, claim that you are working on it. + +Adding a hardcoded function +--------------------------- +The most simple cases are when the existing Abstract Structure :py:class:`pysd.translators.structures.abstract_expressions.CallStructure` can be used. This structure holds a reference to the function name and the passed arguments in a :py:class:`tuple`. Sometimes, the function can be directly added to the model file without the needing of defining any specific function. This can be done when the function is already implemented in Python or in a Python library, and the behaviour is the same. For example, `Vensim's ABS `_ and `XMILE's ABS `_ functions can be replaced by :py:func:`numpy.abs`. + +In this simple case, we only need to include the translation in the :py:data:`functionspace` dictionary from :py:mod:`pysd.builders.python.python_functions.py`:: + + "abs": ("np.abs(%(0)s)", ("numpy",)), + +They key (:py:data:`"abs"`) is the name of the Vensim/XMILE function in lowercase. The first argument in the value (:py:data:`"np.abs(%(0)s)"`) is the Python representation, the :py:data:`%(0)s` stands for the first argument of the original function. The last argument stands for the dependencies of that function; in this case the used function is included in :py:mod:`numpy` module. Hence, we need to import `numpy` in our model file, which is done by adding the dependency :py:data:`("numpy",)`, note that the dependency is a :py:class:`tuple`. + +The next step is to test the new function. In order to do that, we need to include integration tests in the `test-models repo `_. Please, follow the instructions to add a new test in the `README of that repo `_. For this example, we would need to add test models for a Vensim's `mdl` file and a XMILE file, as we are adding support for both. In addition, the tests should cover all the possible cases. For that reason, we should test the absolute of positive and negative floats and positive, negative and mixed arrays. We included the tests `test-models/tests/abs/test_abs.mdl` and `test-models/tests/abs/test_abs.xmile`, with their corresponding outputs file. Now we include the test in the testing script. We need to add the following entry in the :py:data:`vensim_test` dictionary of :py:mod:`tests/pytest_integration/pytest_integration_test_vensim_pathway.py`:: + + "abs": { + "folder": "abs", + "file": "test_abs.mdl" + }, + +and the following one in the :py:data:`xmile_test` dictionary of :py:mod:`tests/pytest_integration/pytest_integration_test_xmile_pathway.py`:: + + "abs": { + "folder": "abs", + "file": "test_abs.xmile" + }, + +At this point we should be able to run the test and, if the implementation was done correctly, they should pass. We also need to make sure that we did not break any other feature by running all the tests. + +In order to finish the contribution, we should update the documentation. The tables of :ref:`supported Vensim functions `, :ref:`supported Xmile functions `, and :ref:`supported Python functions ` are automatically generated from `docs/tables/*.tab`, which are tab separated files. In this case, we should add the following line to `docs/tables/functions.tab`: + +.. list-table:: ABS + :header-rows: 1 + + * - Vensim + - Vensim example + - Xmile + - Xmile example + - Abstract Syntax + - Python Translation + * - ABS + - ABS(A) + - abs + - abs(A) + - CallStructure('abs', (A,)) + - numpy.abs(A) + +To finish, we create a new release notes block at the top of `docs/whats_new.rst` file and update the software version. Commit all the changes, includying the test-models repo, and open a new PR. + + +Adding a simple function +------------------------ +Sometimes, it would be preferable to define own Python functions. This could help to keep similar grammar to the source code, making the final model file content simpler. This example focus on a function where we are still able to use the Abstract Structure :py:class:`pysd.translators.structures.abstract_expressions.CallStructure`, but we will include a function defined in :py:mod:`pysd.py_backend.functions`. + +Let's suppose we want to add support for `Vensim's VECTOR SORT ORDER function `_. First of all, we may need to check Vensim's documentation to see how this function works and try to think what is the fatest way to solve it. VECTOR SORT ORDER function takes two arguments, `vector` and `direction`. The function returns the order of the elements of the `vector` based on the `direction`. Therefore, we do not need to save previous states information or to pass other information as arguments, we should have enough with a basic Python function that takes the same arguments. + +Then, we define the Python function based on the Vensim's documentation. We also include the docstring (with the same style as other functions) and add this function to the file :py:mod:`pysd.py_backend.functions`:: + + + def vector_sort_order(vector, direction): + """ + Implements Vensim's VECTOR SORT ORDER function. Sorting is done on + the complete vector relative to the last subscript. + https://www.vensim.com/documentation/fn_vector_sort_order.html + + Parameters + ----------- + vector: xarray.DataArray + The vector to sort. + direction: float + The direction to sort the vector. If direction > 1 it will sort + the vector entries from smallest to biggest, otherwise from + biggest to smallest. + + Returns + ------- + vector_sorted: xarray.DataArray + The sorted vector. + + """ + if direction <= 0: + flip = np.flip(vector.argsort(), axis=-1) + return xr.DataArray(flip.values, vector.coords, vector.dims) + return vector.argsort() + +Now, we need to link the defined function with its corresponent abstract representation. So we include the following entry in the :py:data:`functionspace` dictionary from :py:mod:`pysd.builders.python.python_functions.py`:: + + "vector_sort_order": ( + "vector_sort_order(%(0)s, %(1)s)", + ("functions", "vector_sort_order")) + +They key (:py:data:`"vector_sort_order"`) is the name of the Vensim function in lowercase and replacing the whitespaces by underscores. The first argument in the value (:py:data:`"vector_sort_order(%(0)s, %(1)s)"`) is the Python representation, the :py:data:`%(0)s` and :py:data:`%(1)s` stand for the first and second argument of the original function, respectively. In this example, the representation is quite similar to the one in Vensim, we will move from `VECTOR SORT ORDER(vec, direction)` to `vector_sort_order(vec, direction)`. The last argument stands for the dependencies of that function; in this case the function has been included in the functions submodule. Hence, we need to import `vector_sort_order` from `functions`, which is done by adding the dependency :py:data:`("functions", "vector_sort_order")`. + +The next step is to add a test model for Vensim's `mdl` file. As the test should cover all the possible cases, we should test the results for one and more dimensions arrays with different values along dimensions to generate different order combinations. Moreover, we have also included cases for the both possible directions. We included the test `test-models/tests/vector_order/test_vector_order.mdl`, with its corresponding outputs file. Now we include the test in the testing script. We need to add the following entry in the :py:data:`vensim_test` dictionary of :py:mod:`tests/pytest_integration/pytest_integration_test_vensim_pathway.py`:: + + "vector_order": { + "folder": "vector_order", + "file": "test_vector_order.mdl" + }, + +At this point we should be able to run the test and, if the implementation was done correctly, they should pass. We also need to make sure that we did not break any other feature by running all the tests. + +In order to finish the contribution, we should update the documentation by adding the following line to `docs/tables/functions.tab`: + +.. list-table:: VECTOR SORT ORDER + :header-rows: 1 + + * - Vensim + - Vensim example + - Xmile + - Xmile example + - Abstract Syntax + - Python Translation + * - VECTOR SORT ORDER + - VECTOR SORT ORDER(vec, direction) + - + - + - CallStructure('vector_sort_order', (vec, direction)) + - vector_sort_order(vec, direction) + +To finish, we create a new release notes block at the top of `docs/whats_new.rst` file and update the software version. Commit all the changes, includying the test-models repo, and open a new PR. diff --git a/docs/development/development_index.rst b/docs/development/development_index.rst index 133ce3bc..aa290022 100644 --- a/docs/development/development_index.rst +++ b/docs/development/development_index.rst @@ -6,9 +6,11 @@ Developer Documentation guidelines pathway + adding_functions pysd_architecture_views/4+1view_model In order to contribut to PySD check the :doc:`guidelines` and the :doc:`pathway`. You also will find helpful the :doc:`Structure of the PySD library <../../structure/structure_index>` to understand better how it works. +If you want to add a missing function to the PySD Python builder you may find useful the example in :doc:`adding_functions`. diff --git a/docs/structure/vensim_translation.rst b/docs/structure/vensim_translation.rst index d1796459..91271eb6 100644 --- a/docs/structure/vensim_translation.rst +++ b/docs/structure/vensim_translation.rst @@ -63,6 +63,7 @@ All the basic operators are supported, this includes the ones shown in the table Moreover, the Vensim :EXCEPT: operator is also supported to manage exceptions in the subscripts. See the :ref:`Subscripts section` section. + Functions ^^^^^^^^^ The list of currentlty supported Vensim functions are detailed below: diff --git a/docs/tables/functions.tab b/docs/tables/functions.tab index 2377741f..8fb198d2 100644 --- a/docs/tables/functions.tab +++ b/docs/tables/functions.tab @@ -1,5 +1,5 @@ Vensim Vensim example Xmile Xmile example Abstract Syntax Python Translation Vensim comments Xmile comments Python comments -ABS ABS(A) abs(A) abs(A) "CallStructure('abs', (A,))" numpy.abs(A) +ABS ABS(A) abs abs(A) "CallStructure('abs', (A,))" numpy.abs(A) MIN "MIN(A, B)" min "min(A, B)" "CallStructure('min', (A, B))" "numpy.minimum(A, B)" MAX "MAX(A, B)" max "max(A, B)" "CallStructure('max', (A, B))" "numpy.maximum(A, B)" SQRT SQRT(A) sqrt sqrt(A) "CallStructure('sqrt', (A,))" numpy.sqrt @@ -33,6 +33,9 @@ PULSE TRAIN PULSE TRAIN(start, width, tbetween, end) "CallStructure('pulse_tra RAMP RAMP(slope, start_time, end_time) ramp ramp(slope, start_time, end_time) "CallStructure('ramp', (slope, start_time, end_time))" pysd.functions.ramp(time, slope, start_time, end_time) Not tested for Xmile! ramp ramp(slope, start_time) "CallStructure('ramp', (slope, start_time))" pysd.functions.ramp(time, slope, start_time) Not tested for Xmile! STEP STEP(height, step_time) step step(height, step_time) "CallStructure('step', (height, step_time))" pysd.functions.step(time, height, step_time) Not tested for Xmile! +VECTOR RANK VECTOR RANK(vec, direction) "CallStructure('vector_rank', (vec, direction))" vector_rank(vec, direction) +VECTOR REORDER VECTOR REORDER(vec, svec) "CallStructure('vector_reorder', (vec, svec))" vector_reorder(vec, svec) +VECTOR SORT ORDER VECTOR SORT ORDER(vec, direction) "CallStructure('vector_sort_order', (vec, direction))" vector_sort_order(vec, direction) GAME GAME(A) GameStructure(A) A INITIAL INITIAL(value) init init(value) InitialStructure(value) pysd.statefuls.Initial SAMPLE IF TRUE "SAMPLE IF TRUE(condition, input, initial_value)" "SampleIfTrueStructure(condition, input, initial_value)" pysd.statefuls.SampleIfTrue(...) diff --git a/docs/whats_new.rst b/docs/whats_new.rst index 06bec077..0dcaeeec 100644 --- a/docs/whats_new.rst +++ b/docs/whats_new.rst @@ -1,6 +1,37 @@ What's New ========== +v3.1.0 (2022/06/02) +------------------- + +New Features +~~~~~~~~~~~~ +- Add support for Vensim's `VECTOR SORT ORDER `_ (:func:`pysd.py_backend.functions.vector_sort_order`) function (:issue:`326`). +- Add support for Vensim's `VECTOR RANK `_ (:func:`pysd.py_backend.functions.vector_rank`) function (:issue:`326`). +- Add support for Vensim's `VECTOR REORDER `_ (:func:`pysd.py_backend.functions.vector_reorder`) function (:issue:`326`). + +Breaking changes +~~~~~~~~~~~~~~~~ + +Deprecations +~~~~~~~~~~~~ + +Bug fixes +~~~~~~~~~ + +Documentation +~~~~~~~~~~~~~ +- Add the section :doc:`/development/adding_functions` with examples for developers. + +Performance +~~~~~~~~~~~ + +Internal Changes +~~~~~~~~~~~~~~~~ + +- Include a template for PR. + + v3.0.1 (2022/05/26) ------------------- diff --git a/pysd/_version.py b/pysd/_version.py index 05527687..f5f41e56 100644 --- a/pysd/_version.py +++ b/pysd/_version.py @@ -1 +1 @@ -__version__ = "3.0.1" +__version__ = "3.1.0" diff --git a/pysd/builders/python/python_functions.py b/pysd/builders/python/python_functions.py index e8560205..f8201deb 100644 --- a/pysd/builders/python/python_functions.py +++ b/pysd/builders/python/python_functions.py @@ -81,6 +81,15 @@ "zidz": ( "zidz(%(0)s, %(1)s)", ("functions", "zidz")), + "vector_sort_order": ( + "vector_sort_order(%(0)s, %(1)s)", + ("functions", "vector_sort_order")), + "vector_reorder": ( + "vector_reorder(%(0)s, %(1)s)", + ("functions", "vector_reorder")), + "vector_rank": ( + "vector_rank(%(0)s, %(1)s)", + ("functions", "vector_rank")), # random functions must have the shape of the component subscripts # most of them are shifted, scaled and truncated diff --git a/pysd/builders/python/python_model_builder.py b/pysd/builders/python/python_model_builder.py index 1f6e379f..2d013706 100644 --- a/pysd/builders/python/python_model_builder.py +++ b/pysd/builders/python/python_model_builder.py @@ -67,8 +67,8 @@ def build_model(self) -> Path: class SectionBuilder: """ SectionBuilder allows building a section of the PySD model. Each - section will be a file unless the model has been setted to be - split in modules. + section will be a file unless the model has been set to be split + in modules. Parameters ---------- diff --git a/pysd/py_backend/functions.py b/pysd/py_backend/functions.py index 05349ffe..f019ae37 100644 --- a/pysd/py_backend/functions.py +++ b/pysd/py_backend/functions.py @@ -11,6 +11,8 @@ import numpy as np import xarray as xr +from . import utils + small_vensim = 1e-6 # What is considered zero according to Vensim Help @@ -206,7 +208,7 @@ def xidz(numerator, denominator, x): def zidz(numerator, denominator): """ This function bypasses divide-by-zero errors, - implementing Vensim's ZIDZ function + implementing Vensim's ZIDZ function. https://www.vensim.com/documentation/fn_zidz.htm Parameters @@ -471,3 +473,107 @@ def invert_matrix(mat): # NUMPY: avoid converting to xarray, put directly the expression # in the model return xr.DataArray(np.linalg.inv(mat.values), mat.coords, mat.dims) + + +def vector_sort_order(vector, direction): + """ + Implements Vensim's VECTOR SORT ORDER function. Sorting is done on + the complete vector relative to the last subscript. + https://www.vensim.com/documentation/fn_vector_sort_order.html + + Parameters + ----------- + vector: xarray.DataArray + The vector to sort. + direction: float + The direction to sort the vector. If direction > 1 it will sort + the vector entries from smallest to biggest, otherwise from + biggest to smallest. + + Returns + ------- + vector_sorted: xarray.DataArray + The sorted vector. + + """ + # TODO: can direction be an array? In this case this will fail + if direction <= 0: + # NUMPY: return flip directly + flip = np.flip(vector.argsort(), axis=-1) + return xr.DataArray(flip.values, vector.coords, vector.dims) + return vector.argsort() + + +def vector_reorder(vector, svector): + """ + Implements Vensim's VECTOR REORDER function. Reordering is done on + the complete vector relative to the last subscript. + https://www.vensim.com/documentation/fn_vector_reorder.html + + Parameters + ----------- + vector: xarray.DataArray + The vector to sort. + svector: xarray.DataArray + The vector to specify the order. + + Returns + ------- + vector_sorted: xarray.DataArray + The sorted vector. + + """ + # NUMPY: Use directly numpy sort functions, no need to assign coords later + if len(svector.dims) > 1: + # TODO this may be simplified + new_vector = vector.copy() + dims = svector.dims + # create an empty array to hold the orderings (only last dim) + arr = xr.DataArray( + np.nan, + {dims[-1]: vector.coords[dims[-1]].values}, + dims[-1:] + ) + # split the ordering array in 0-dim arrays + svectors = utils.xrsplit(svector) + orders = {} + for sv in svectors: + # regrup the ordering arrays using last dimensions + pos = {dim: str(sv.coords[dim].values) for dim in dims[:-1]} + key = ";".join(pos.values()) + if key not in orders.keys(): + orders[key] = (pos, arr.copy()) + orders[key][1].loc[sv.coords[dims[-1]]] = sv.values + + for pos, array in orders.values(): + # get the reordered array + values = [vector.loc[pos].values[int(i)] for i in array.values] + new_vector.loc[pos] = values + + return new_vector + + return vector[svector.values].assign_coords(vector.coords) + + +def vector_rank(vector, direction): + """ + Implements Vensim's VECTOR RANK function. Ranking is done on the + complete vector relative to the last subscript. + https://www.vensim.com/documentation/fn_vector_rank.html + + Parameters + ----------- + vector: xarray.DataArray + The vector to sort. + direction: float + The direction to sort the vector. If direction > 1 it will rank + the vector entries from smallest to biggest, otherwise from + biggest to smallest. + + Returns + ------- + vector_rank: xarray.DataArray + The rank of the vector. + + """ + return vector_sort_order(vector, direction).argsort() + 1 diff --git a/tests/pytest_integration/pytest_integration_vensim_pathway.py b/tests/pytest_integration/pytest_integration_vensim_pathway.py index f7dbc389..b3f2eb7c 100644 --- a/tests/pytest_integration/pytest_integration_vensim_pathway.py +++ b/tests/pytest_integration/pytest_integration_vensim_pathway.py @@ -509,6 +509,10 @@ "folder": "variable_ranges", "file": "test_variable_ranges.mdl" }, + "vector_order": { + "folder": "vector_order", + "file": "test_vector_order.mdl" + }, "xidz_zidz": { "folder": "xidz_zidz", "file": "xidz_zidz.mdl" @@ -580,4 +584,4 @@ def test_read_vensim_file(self, model_path, data_path, kwargs): with warnings.catch_warnings(): warnings.simplefilter("ignore") output, canon = runner(model_path, data_files=data_path) - assert_frames_close(output, canon, **kwargs) + assert_frames_close(output, canon, verbose=True, **kwargs) diff --git a/tests/test-models b/tests/test-models index 1502dce4..69257807 160000 --- a/tests/test-models +++ b/tests/test-models @@ -1 +1 @@ -Subproject commit 1502dce4b5dbe8d86e6f310fc40f5c33a6dea1ec +Subproject commit 692578073bbc22bf140175573da41fbcd060b612