Merge pull request #300 from enekomartinmartinez/run_view

Add Data class and allow runing models with regular Data declaration. 
Allow running a submodels.

docs/advanced_usage.rst

@@ -66,7 +66,8 @@ In a Vensim model with three separate views (e.g. `view_1`, `view_2` and `view_3
 | │   ├── view_2.py
 | │   └── view_3.py
 | ├── _namespace_many_views_model.json
-| ├── _subscripts_dict_many_views_model.json
+| ├── _subscripts_many_views_model.json
+| ├── _dependencies_many_views_model.json
 | ├── many_views_model.py
 |
 |
@@ -109,3 +110,16 @@ the new simulation will have initial time equal to 50 with the saved values from
 .. warning::
   Exported data is saved and loaded using `pickle <https://docs.python.org/3/library/pickle.html>`_, this data can be incompatible with future versions of
   *PySD* or *xarray*. In order to prevent data losses save always the source code.
+
+
+Selecting and running a submodel
+--------------------------------
+A submodel of a translated model can be selected in order to run only a part of the original model. This can be done through the :py:data:`.select_submodel()` method:
+
+.. autoclass:: pysd.py_backend.statefuls.Model
+   :members: select_submodel
+
+In order to preview the needed exogenous variables the :py:data:`.get_dependencies()` method can be used:
+
+.. autoclass:: pysd.py_backend.statefuls.Model
+   :members: get_dependencies

docs/basic_usage.rst

@@ -58,6 +58,27 @@ To show a progressbar during the model integration the progress flag can be pass
 
    >>> stocks = model.run(progress=True)
 
+Running models with DATA type components
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Venim's regular DATA type components are given by an empty expression in the model equation. These values are read from a binary `.vdf` file. PySD allows running models with this kind of data definition using the data_files argument when calling :py:func:`.run()` command, e.g.::
+
+   >>> stocks = model.run(data_files="input_data.tab")
+
+Several files can be passed by using a list, then if the data information has not been found in the first file, the next one will be used until finding the data values::
+
+   >>> stocks = model.run(data_files=["input_data.tab", "input_data2.tab", ..., "input_datan.tab"])
+
+If a variable is given in different files to choose the specific file a dictionary can be used::
+
+   >>> stocks = model.run(data_files={"input_data.tab": ["data_var1", "data_var3"], "input_data2.tab": ["data_var2"]})
+
+.. note::
+   Only `tab` and `csv` files are supported, they should be given as a table, each variable one column (or row) and the time in the first column (or first row). The column (or row) names can be given using the original name or using python names.
+
+.. note::
+   Subscripted variables must be given in the vensim format, one column (or row) per subscript combination. Example of column names for 2x2 variable:
+      `subs var[A, C]`  `subs var[B, C]`  `subs var[A, D]`  `subs var[B, D]`
+
 Outputting various run information
 ----------------------------------
 The :py:func:`.run()` command has a few options that make it more useful. In many situations we want to access components of the model other than merely the stocks – we can specify which components of the model should be included in the returned dataframe by including them in a list that we pass to the :py:func:`.run()` command, using the return_columns keyword argument::

docs/development/internal_functions.rst

@@ -29,7 +29,7 @@ Special functions needed for model execution
 Building the python model file
 ------------------------------
 
-.. automodule:: pysd.py_backend.builder
+.. automodule:: pysd.translation.builder
    :members:
    :undoc-members:
    :private-members:

docs/development/vensim_translation.rst

@@ -34,7 +34,7 @@ parsed, and may throw an error. Future releases will handle this with more grace
 Used Functions for Translation
 ------------------------------
 
-.. automodule:: pysd.py_backend.vensim.vensim2py
+.. automodule:: pysd.translation.vensim.vensim2py
    :members:
    :undoc-members:
    :private-members:

pysd/_version.py

@@ -1 +1 @@
-__version__ = "2.0.0"
+__version__ = "2.1.0"

pysd/cli/main.py

@@ -1,6 +1,7 @@
 import sys
 import os
 
+from csv import QUOTE_NONE
 from datetime import datetime
 
 from .parser import parser
@@ -25,8 +26,9 @@ def main(args):
     """
     options = parser.parse_args(args)
 
-    model = load(options.model_file, options.missing_values,
-                 options.split_views, subview_sep=options.subview_sep)
+    model = load(options.model_file, options.data_files,
+                 options.missing_values, options.split_views,
+                 subview_sep=options.subview_sep)
 
     if not options.run:
         print("\nFinished!")
@@ -44,7 +46,7 @@ def main(args):
     sys.exit()
 
 
-def load(model_file, missing_values, split_views, **kwargs):
+def load(model_file, data_files, missing_values, split_views, **kwargs):
     """
     Translate and load model file.
 
@@ -53,6 +55,18 @@ def load(model_file, missing_values, split_views, **kwargs):
     model_file: str
         Vensim, Xmile or PySD model file.
 
+    data_files: list
+        If given the list of files where the necessary data to run the model
+        is given.
+
+    missing_values : str ("warning", "error", "ignore", "keep")
+        What to do with missing values. If "warning" (default)
+        shows a warning message and interpolates the values.
+        If "raise" raises an error. If "ignore" interpolates
+        the values without showing anything. If "keep" it will keep
+        the missing values, this option may cause the integration to
+        fail, but it may be used to check the quality of the data.
+
     split_views: bool (optional)
         If True, the sketch is parsed to detect model elements in each
         model view, and then translate each view in a separate python
@@ -74,14 +88,17 @@ def load(model_file, missing_values, split_views, **kwargs):
     if model_file.lower().endswith(".mdl"):
         print("\nTranslating model file...\n")
         return pysd.read_vensim(model_file, initialize=False,
+                                data_files=data_files,
                                 missing_values=missing_values,
                                 split_views=split_views, **kwargs)
     elif model_file.lower().endswith(".xmile"):
         print("\nTranslating model file...\n")
         return pysd.read_xmile(model_file, initialize=False,
+                               data_files=data_files,
                                missing_values=missing_values)
     else:
         return pysd.load(model_file, initialize=False,
+                         data_files=data_files,
                          missing_values=missing_values)
 
 
@@ -147,6 +164,8 @@ def save(output, options):
     else:
         sep = ","
 
-    output.to_csv(output_file, sep, index_label="Time")
+    # QUOTE_NONE used to print the csv/tab files af vensim does with special
+    # characterse, e.g.: "my-var"[Dimension]
+    output.to_csv(output_file, sep, index_label="Time", quoting=QUOTE_NONE)
 
     print(f"Data saved in '{output_file}'")

pysd/cli/parser.py

@@ -54,6 +54,32 @@ def check_model(string):
     return string
 
 
+def check_data_file(string):
+    """
+    Check that data file is a tab or csv file and that exists.
+    """
+    if not string.endswith('.tab') and not string.endswith('.csv'):
+        parser.error(
+            f'when parsing {string}'
+            '\nThe data file name must be .tab or .csv...')
+    elif not os.path.isfile(string):
+        parser.error(
+            f'when parsing {string}'
+            '\nThe data file does not exist...')
+    else:
+        return string
+
+
+def split_files(string):
+    """
+    Splits the data files and returns and error if file doesn't exists
+    --data 'file1.tab, file2.csv' -> ['file1.tab', 'file2.csv']
+    --data file1.tab -> ['file1.tab']
+
+    """
+    return [check_data_file(s.strip()) for s in string.split(',')]
+
+
 def split_columns(string):
     """
     Splits the return-columns argument or reads it from .txt
@@ -84,6 +110,15 @@ def split_timestamps(string):
             f'See {docs} for examples.')
 
 
+def split_subview_sep(string):
+    """
+    Splits the subview separators
+    --subview-sep ' - ,.' -> [' - ', '.']
+
+    """
+    return string.split(",")
+
+
 def split_vars(string):
     """
     Splits the arguments from new_values.
@@ -217,6 +252,10 @@ def __call__(self, parser, namespace, values, option_string=None):
     help='provide the return time stamps separated by commas, if given '
          '--saveper will be ignored')
 
+model_arguments.add_argument(
+    '-D', '--data', dest='data_files',
+    action='store', type=split_files, metavar='\'FILE1, FILE2, .., FILEN\'',
+    help='input data file or files to run the model')
 
 #########################
 # Translation arguments #
@@ -240,7 +279,8 @@ def __call__(self, parser, namespace, values, option_string=None):
 
 trans_arguments.add_argument(
     '--subview-sep', dest='subview_sep',
-    action='store', type=str, default="", metavar='STRING',
+    action='store', type=split_subview_sep, default=[],
+    metavar='\'STRING1,STRING2,..,STRINGN\'',
     help='further division of views split in subviews, by identifying the'
          'separator string in the view name, only availabe if --split-views'
          ' is used')

pysd/py_backend/components.py

@@ -142,7 +142,10 @@ def in_return(self):
         if self.return_timestamps is not None:
             return self._time in self.return_timestamps
 
-        return (self._time - self._initial_time) % self.saveper() == 0
+        time_delay = self._time - self._initial_time
+        save_per = self.saveper()
+        prec = self.time_step() * 1e-10
+        return time_delay % save_per < prec or -time_delay % save_per < prec
 
     def add_return_timestamps(self, return_timestamps):
         """ Add return timestamps """