Merge pull request #141 from gmdsi/feat_dsi

Feat dsi

dependencies/pyemu/CITATION.cff

@@ -3,8 +3,8 @@ message: If you use this software, please cite both the article from preferred-c
   and the software itself.
 type: software
 title: pyEMU
-version: 1.3.4
-date-released: '2024-03-27'
+version: 1.3.6
+date-released: '2024-06-17'
 abstract: A python package for interfacing with PEST and PEST++, and providing model-independent data-worth, and linear and non-linear uncertainty analyses.
 repository-artifact: https://pypi.org/project/pyemu
 repository-code: https://github.com/pypest/pyemu

dependencies/pyemu/pyemu/_version.py

@@ -26,9 +26,9 @@ def get_keywords():
     # setup.py/versioneer.py will grep for the variable names, so they must
     # each be defined on a line of their own. _version.py will just call
     # get_keywords().
-    git_refnames = "$Format:%d$"
-    git_full = "$Format:%H$"
-    git_date = "$Format:%ci$"
+    git_refnames = " (HEAD -> develop)"
+    git_full = "5551e161bb4978c599b7352ee373b0d88d831d68"
+    git_date = "2024-07-19 20:41:32 +0100"
     keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
     return keywords
 

dependencies/pyemu/pyemu/en.py

@@ -1346,7 +1346,7 @@ def from_parfiles(cls, pst, parfile_names, real_names=None):
                     ),
                     PyemuWarning,
                 )
-                blank_df = pd.DataFrame(index=df_all.index, columns=diff)
+                blank_df = pd.DataFrame(index=df_all.index, columns=list(diff))
 
                 df_all = pd.concat([df_all, blank_df], axis=1)
 
@@ -1697,10 +1697,6 @@ def _enforce_reset(self, bound_tol):
         violating vals to bound
         """
 
-        ub = (self.ubnd * (1.0 - bound_tol)).to_dict()
-        lb = (self.lbnd * (1.0 + bound_tol)).to_dict()
-
-        val_arr = self._df.values
-        for iname, name in enumerate(self.columns):
-            val_arr[val_arr[:, iname] > ub[name], iname] = ub[name]
-            val_arr[val_arr[:, iname] < lb[name], iname] = lb[name]
+        ub = self.ubnd * (1.0 - bound_tol)
+        lb = self.lbnd * (1.0 + bound_tol)
+        self._df = self._df.clip(lb, ub, axis=1)

dependencies/pyemu/pyemu/mat/mat_handler.py

@@ -2098,7 +2098,7 @@ def to_binary(self, filename, droptol=None, chunk=None):
         f.close()
 
     @staticmethod
-    def read_dense(filename, forgive=False, close=True):
+    def read_dense(filename, forgive=False, close=True, only_rows=None):
         """read a dense-format binary file.
 
         Args:
@@ -2108,6 +2108,7 @@ def read_dense(filename, forgive=False, close=True):
                 records are returned.  If False, an exception is raised for an
                 incomplete record
             close (`bool`): flag to close the filehandle.  Default is True
+            only_rows (`iterable`): rows to read.  If None, all rows are read
 
         Returns:
             tuple containing
@@ -2126,27 +2127,26 @@ def read_dense(filename, forgive=False, close=True):
             f = open(filename, "rb")
         else:
             f = filename
-        # the header datatype
-        itemp1, itemp2, icount = np.fromfile(f, Matrix.binary_header_dt, 1)[0]
-        # print(itemp1,itemp2,icount)
-        if itemp1 != 0:
-            raise Exception("Matrix.read_dense() itemp1 != 0")
-        if itemp2 != icount:
-            raise Exception("Matrix.read_dense() itemp2 != icount")
-        ncol = np.abs(itemp2)
+
         col_names = []
         row_names = []
         data_rows = []
-        col_slens = np.fromfile(f, Matrix.integer, ncol)
+
+        row_names,row_offsets,col_names,success = Matrix.get_dense_binary_info(filename)
+        if not forgive and not success:
+            raise Exception("Matrix.read_dense(): error reading dense binary info")
+        if only_rows is not None:
+            missing = list(set(only_rows)-set(row_names))
+            if len(missing) > 0:
+                raise Exception("the following only_rows are missing:{0}".format(",".join(missing)))
+            only_offsets = [row_offsets[row_names.index(only_row)] for only_row in only_rows]
+            row_names = only_rows
+            row_offsets = only_offsets
+        ncol = len(col_names)
+
         i = 0
-        # for j in range(ncol):
-        for slen in col_slens:
-            # slen = np.fromfile(f, Matrix.integer,1)[0]
-            name = (
-                struct.unpack(str(slen) + "s", f.read(slen))[0].strip().lower().decode()
-            )
-            col_names.append(name)
-        while True:
+        for row_name,offset in zip(row_names,row_offsets):
+            f.seek(offset)
             try:
                 slen = np.fromfile(f, Matrix.integer, 1)[0]
             except Exception as e:
@@ -2172,7 +2172,6 @@ def read_dense(filename, forgive=False, close=True):
                     break
                 else:
                     raise Exception("error reading row {0}: {1}".format(i, str(e)))
-            row_names.append(name)
             data_rows.append(data_row)
             i += 1
 
@@ -2181,6 +2180,91 @@ def read_dense(filename, forgive=False, close=True):
             f.close()
         return data_rows, row_names, col_names
 
+
+
+    @staticmethod
+    def get_dense_binary_info(filename):
+        """read the header and row and offsets for a dense binary file.
+
+        Parameters
+        ----------
+            fileanme (`str`): dense binary filename
+
+
+        Returns:
+            tuple containing
+
+            - **['str']**: list of row names
+            - **['int']**: list of row offsets
+            - **[`str`]**: list of col names
+            - **bool**: flag indicating successful reading of all records found
+
+
+        """
+        if not os.path.exists(filename):
+            raise Exception(
+                "Matrix.read_dense(): filename '{0}' not found".format(filename)
+            )
+        if isinstance(filename, str):
+            f = open(filename, "rb")
+        else:
+            f = filename
+        # the header datatype
+        itemp1, itemp2, icount = np.fromfile(f, Matrix.binary_header_dt, 1)[0]
+        # print(itemp1,itemp2,icount)
+        if itemp1 != 0:
+            raise Exception("Matrix.read_dense() itemp1 != 0")
+        if itemp2 != icount:
+            raise Exception("Matrix.read_dense() itemp2 != icount")
+        ncol = np.abs(itemp2)
+        col_slens = np.fromfile(f, Matrix.integer, ncol)
+        i = 0
+        col_names = []
+        for slen in col_slens:
+            name = (
+                struct.unpack(str(slen) + "s", f.read(slen))[0].strip().lower().decode()
+            )
+            col_names.append(name)
+        row_names = []
+        row_offsets = []
+        data_len = np.array(1,dtype=Matrix.double).itemsize * ncol
+        success = True
+        while True:
+            curr_pos = f.tell()
+            try:
+                slen = np.fromfile(f, Matrix.integer, 1)[0]
+            except Exception as e:
+                break
+            try:
+                name = (
+                    struct.unpack(str(slen) + "s", f.read(slen))[0]
+                    .strip()
+                    .lower()
+                    .decode()
+                )
+            except Exception as e:
+                    print("error reading row name {0}: {1}".format(i, str(e)))
+                    success = False
+                    break
+            try:
+                data_row = np.fromfile(f, Matrix.double, ncol)
+                if data_row.shape[0] != ncol:
+                    raise Exception(
+                        "incomplete data in row {0}: {1} vs {2}".format(
+                            i, data_row.shape[0], ncol))
+            except Exception as e:
+                print("error reading row data record {0}: {1}".format(i, str(e)))
+                success = False
+                break
+
+            row_offsets.append(curr_pos)
+            row_names.append(name)
+
+            i += 1
+        f.close()
+        return row_names,row_offsets,col_names,success
+
+
     @classmethod
     def from_binary(cls, filename, forgive=False):
         """class method load from PEST-compatible binary file into a

dependencies/pyemu/pyemu/plot/plot_utils.py

@@ -363,7 +363,7 @@ def res_1to1(
         obs_g.loc[:, "sim"] = res.loc[names, "modelled"]
         logger.statement("using control file obsvals to calculate residuals")
         obs_g.loc[:, "res"] = obs_g.sim - obs_g.obsval
-        if "include_zero" not in kwargs or kwargs["include_zero"] is True:
+        if "include_zero" not in kwargs or kwargs["include_zero"] is False:
             obs_g = obs_g.loc[obs_g.weight > 0, :]
         if obs_g.shape[0] == 0:
             logger.statement("no non-zero obs for group '{0}'".format(g))

dependencies/pyemu/pyemu/pst/pst_handler.py

@@ -7,10 +7,8 @@
 import time
 import warnings
 import numpy as np
-from numpy.lib.type_check import real_if_close
 import pandas as pd
 
-
 pd.options.display.max_colwidth = 100
 # pd.options.mode.use_inf_as_na = True
 import pyemu
@@ -21,6 +19,7 @@
 
 # from pyemu.utils.os_utils import run
 
+
 def get_constraint_tags(ltgt='lt'):
     if ltgt == 'lt':
         return "l_", "less", ">@"
@@ -2290,9 +2289,7 @@ def parrep(
                     real_name, parfile
                 )
             )
-            self.parameter_data.parval1 = parens.loc[real_name].T.loc[
-                self.parameter_data.parnme
-            ]
+            self.parameter_data["parval1"] = parens.loc[real_name,self.par_names]
 
         if enforce_bounds:
             par = self.parameter_data