update 1.2.4

Documentation/EGADS Algorithm Handbook - LATEX/cover/alg_cover.tex

@@ -24,7 +24,7 @@
 \setlength{\voffset}{.0cm}
 
 \logoeufar
-\title{EUFAR, Olivier Henry}{}{EGADS Lineage Algorithm Handbook}{Version 1.2.3}
+\title{EUFAR, Olivier Henry}{}{EGADS Lineage Algorithm Handbook}{Version 1.2.4}
 %\author{}{}
 
 

Documentation/changelog.txt

@@ -6,11 +6,24 @@ To do list:
 -----------
   - check feasability of an online algorithm database.
   - add function to create algorithm template.
+  - add function to add a folder, somewhere on the hard drive, to the list of algorithms.
 
 To fix list:
 ------------
 
 
+April 20 2020, Release version 1.2.4
+====================================
+  FIXED:
+    - corrected mistakes in the documentation (mostly python 2 old code).
+
+  MODIFIED:
+    - the read_variable function in hdf_io.py has been modified, now if a missing value attribute is missing, a warning is displayed in the log, but no exception is raised.
+
+  ADDED:
+    - the get_group_list function has been added to hdf_io.py
+
+
 April 1 2020, Release version 1.2.3
 ===================================
   FIXED:

PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: egads-lineage
-Version: 1.2.3
+Version: 1.2.4
 Summary: EGADS: EUFAR General Airborne Data-processing Software
 Home-page: https://github.com/EUFAR/egads/tree/Lineage
 Author: EUFAR, Olivier Henry

README.md

@@ -1,7 +1,7 @@
 Version:
 -------
 
-EGADS Lineage 1.2.3
+EGADS Lineage 1.2.4
 
 
 Developments:

doc/source/conf.py

@@ -48,9 +48,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.2.3'
+version = '1.2.4'
 # The full version, including alpha/beta/rc tags.
-release = '1.2.3'
+release = '1.2.4'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/source/tutorial-csv.inc

@@ -136,12 +136,12 @@ This file has been created with the following commands:
 
 * create two main variables, following the official EGADS convention:
 
-    >>> data1 = egads.EgadsData(value=[5.0,2.0,-2.0,0.5,4.0], units='mm', name='sea level', scale_factor=1., add_offset=0., _FillValue=-9999)
-    >>> data2 = egads.EgadsData(value=[1.0,3.0,-1.0,2.5,6.0], units='mm', name='corr sea level', scale_factor=1., add_offset=0., _FillValue=-9999)
+    >>> data1 = egads.EgadsData(value=[5.0,2.0,-2.0,0.5,4.0], units='mm', standard_name='sea level', scale_factor=1., add_offset=0., _FillValue=-9999)
+    >>> data2 = egads.EgadsData(value=[1.0,3.0,-1.0,2.5,6.0], units='mm', standard_name='corr sea level', scale_factor=1., add_offset=0., _FillValue=-9999)
 
 * create an independant variable, still by following the official EGADS convention:
 
-    >>> time = egads.EgadsData(value=[1.0,2.0,3.0,4.0,5.0], units='seconds since 19700101T00:00:00', name='time')
+    >>> time = egads.EgadsData(value=[1.0,2.0,3.0,4.0,5.0], units='seconds since 19700101T00:00:00', standard_name='time')
 
 * create a new EgadsFile instance:
 

doc/source/tutorial-end.inc

@@ -24,7 +24,7 @@ Within Python, usage information on each algorithm can be found using the ``help
 	class VelocityTasCnrm(egads.core.egads_core.EgadsAlgorithm)
 	 |  FILE        velocity_tas_cnrm.py
 	 | 
-	 |  VERSION     $Revision: 104 $
+	 |  VERSION     Revision: 1.02
 	 |  
 	 |  CATEGORY    Thermodynamics
 	 |  

doc/source/tutorial-global.rst

@@ -88,9 +88,9 @@ If, during the call to :class:`~.EgadsData`, no units are provided, but a variab
 
    >>> x_metadata = egads.core.metadata.VariableMetadata({'units':'m', 'long_name':'Test Variable'})
    >>> x = egads.EgadsData([1,2,3], x_metadata)
-   >>> print x.units
+   >>> print(x.units)
    m
-   >>> print x.metadata
+   >>> print(x.metadata)
    {'units': 'm', 'long_name': 'Test Variable'}
 
 The :class:`~.EgadsData` is a subclass of the Quantities and Numpy packages. Thus it allows different kind of operations like addition, substraction, slicing, and many more. For each of those operations, a new :class:`~.EgadsData` class is created with all their attributes. 
@@ -132,7 +132,7 @@ Once a unit type has been assigned to an :class:`~.EgadsData` instance, it will
 
    >>> a = egads.EgadsData([1,2,3], 'm')
    >>> a_km = a.rescale('km')
-   >>> print a_km
+   >>> print(a_km)
    ['EgadsData', array([0.001, 0.002, 0.003]), 'km']
    >>> a_grams = a.rescale('g')
    ValueError: Unable to convert between units of "m" and "g"

doc/source/tutorial-hdf.inc

@@ -83,6 +83,19 @@ Getting info
 
    >>> print(f.get_file_structure())
 
+.. function:: f.get_group_list([groupname=None, details=False])
+
+   Returns a list of groups found in the Hdf file. If *groupname* is provided, the function returns all groups from *groupname* to the last group in *groupname*. The function returns a list of string if details is False. If details is True, it returns a list of dictionary in which the key is the name of the group and the value its path in the Hdf file.
+
+   :param groupname: name of a group where to get the group list ; optional
+   :type groupname: string
+   :param details: if True, the function returns a list of dictionary in which the key is the name of the group and the value its path in the Hdf file ; optional
+   :type details: bool
+   :rtype: list of strings or list of dictionary
+
+   >>> print(f.get_group_list())
+   >>> print(f.get_group_list('test1', True))
+
 .. function:: f.get_filename()
 
    Returns the filename for the currently opened file.

doc/source/tutorial-netcdf.inc

@@ -300,12 +300,12 @@ This file has been created with the following commands:
 
 * create two main variables, following the official EGADS convention:
 
-    >>> data1 = egads.EgadsData(value=[5.0,2.0,-2.0,0.5,4.0], units='mm', name='sea level', scale_factor=1., add_offset=0., _FillValue=-9999)
-    >>> data2 = egads.EgadsData(value=[1.0,3.0,-1.0,2.5,6.0], units='mm', name='corr sea level', scale_factor=1., add_offset=0., _FillValue=-9999)
+    >>> data1 = egads.EgadsData(value=[5.0,2.0,-2.0,0.5,4.0], units='mm', standard_name='sea level', scale_factor=1., add_offset=0., _FillValue=-9999)
+    >>> data2 = egads.EgadsData(value=[1.0,3.0,-1.0,2.5,6.0], units='mm', standard_name='corr sea level', scale_factor=1., add_offset=0., _FillValue=-9999)
 
 * create an independant variable, still by following the official EGADS convention:
 
-    >>> time = egads.EgadsData(value=[1.0,2.0,3.0,4.0,5.0], units='seconds since 19700101T00:00:00', name='time')
+    >>> time = egads.EgadsData(value=[1.0,2.0,3.0,4.0,5.0], units='seconds since 19700101T00:00:00', standard_name='time')
 
 * create a new EgadsNetCdf instance with a file name:
 

doc/source/tutorial-raw.inc

@@ -106,12 +106,12 @@ This file has been created with the following commands:
 
 * create two main variables, following the official EGADS convention:
 
-    >>> data1 = egads.EgadsData(value=[5.0,2.0,-2.0,0.5,4.0], units='mm', name='sea level', scale_factor=1., add_offset=0., _FillValue=-9999)
-    >>> data2 = egads.EgadsData(value=[1.0,3.0,-1.0,2.5,6.0], units='mm', name='corr sea level', scale_factor=1., add_offset=0., _FillValue=-9999)
+    >>> data1 = egads.EgadsData(value=[5.0,2.0,-2.0,0.5,4.0], units='mm', standard_name='sea level', scale_factor=1., add_offset=0., _FillValue=-9999)
+    >>> data2 = egads.EgadsData(value=[1.0,3.0,-1.0,2.5,6.0], units='mm', standard_name='corr sea level', scale_factor=1., add_offset=0., _FillValue=-9999)
 
 * create an independant variable, still by following the official EGADS convention:
 
-    >>> time = egads.EgadsData(value=[1.0,2.0,3.0,4.0,5.0], units='seconds since 19700101T00:00:00', name='time')
+    >>> time = egads.EgadsData(value=[1.0,2.0,3.0,4.0,5.0], units='seconds since 19700101T00:00:00', standard_name='time')
 
 * create a new EgadsFile instance:
 
@@ -124,7 +124,7 @@ This file has been created with the following commands:
 * prepare the headers if necessary:
 
     >>> headers = '# The current file has been created with EGADS\n# Institution: My Institution\n# Author(s): John Doe\n'
-    >>> headers += time.metadata["long_name"] + '    ' + data1.metadata["long_name"] + '    ' + data2.metadata["long_name"] + '\n''My institution')
+    >>> headers += time.metadata["standard_name"] + '    ' + data1.metadata["standard_name"] + '    ' + data2.metadata["standard_name"])
 
 * prepare an object to receive all data:
 

egads/_version.py

@@ -1,2 +1,2 @@
-__version__ = '1.2.3'
+__version__ = '1.2.4'
 __branch__ = 'Lineage'

egads/input/hdf_io.py

@@ -1,6 +1,6 @@
 __author__ = "ohenry"
 __date__ = "2019-11-22 07:09"
-__version__ = "0.9"
+__version__ = "1.0"
 __all__ = ["Hdf", "EgadsHdf"]
 
 
@@ -49,14 +49,34 @@ def open(self, filename, perms=None):
         logging.debug('egads - hdf_io.py - Hdf - open')
         FileCore.open(self, filename, perms)
 
-    def get_file_structure(self):
+    def get_file_structure(self, from_group=None):
         """
         Returns a view of the file structure, groups and datasets.
 
+        :param str from_group:
+            if from_group is provided, returs file structure from the group from_group.
         :return: file structure.
         """
 
-        return self._get_file_structure()
+        return self._get_file_structure(from_group=from_group)
+
+    def get_group_list(self, groupname=None, details=False):
+        """
+        Returns a list of groups found in the current Hdf file.
+
+        :param string groupname:
+            Optional - the name of the group to get the list from. It should represent a path to
+            the group. None by default.
+        :param bool details:
+            If details is true, it will return a list of all groups in the Hdf file, or from
+            groupname if groupname is not None, and their path. In that case, each element of
+            the list is a small dict containing as key/value the name of the group and the path
+            of the group in the file. False by default.
+        :return: list of groups.
+        """
+
+        logging.debug('egads - hdf_io.py - Hdf - get_group_list')
+        return self._get_group_list(groupname, details)
 
     def get_variable_list(self, groupname=None, group_walk=False, details=False):
         """
@@ -350,19 +370,42 @@ def _open_file(self, filename, perms):
             logging.exception('egads - hdf_io.py - Hdf - _open_file - Exception, Unexpected error')
             raise Exception("ERROR: Unexpected error")
 
-    def _get_file_structure(self):
+    def _get_file_structure(self, from_group):
         """
         Private method for getting a view of the file structure, groups and datasets.
         """
 
+        logging.debug('egads - hdf_io.py - Hdf - _get_file_structure')
         file_structure = collections.OrderedDict()
 
         def _h5py_get_file_structure(name, obj):
             file_structure[name] = {'object': obj, 'type': type(obj)}
 
-        self.f.visititems(_h5py_get_file_structure)
+        if from_group is None:
+            self.f.visititems(_h5py_get_file_structure)
+        else:
+            self.f[from_group].visititems(_h5py_get_file_structure)
         return file_structure
 
+    def _get_group_list(self, groupname, details):
+        """
+        Private method for getting list of group names.
+        """
+
+        logging.debug('egads - hdf_io.py - Hdf - _get_group_list')
+        if self.f is not None:
+            group_list = []
+            for key, value in self._get_file_structure(from_group=groupname).items():
+                if isinstance(value['object'], h5py.Group):
+                    if details:
+                        group_list.append({'name': os.path.basename(key), 'path': '/' + key})
+                    else:
+                        group_list.append(os.path.basename(key))
+            return group_list
+        else:
+            logging.error('egads.input.Hdf._get_group_list: AttributeError, No file open')
+            raise AttributeError('No file open')
+
     def _get_variable_list(self, groupname, group_walk, details):
         """
         Private method for getting list of dataset names.
@@ -550,16 +593,17 @@ def _read_variable(self, varname, input_range, read_as_float, replace_fill_value
             var_attrs = self._get_attribute_list(varname)
             if '_FillValue' in var_attrs:
                 _fill_value = var_attrs['_FillValue']
+                var[var == _fill_value] = numpy.nan
             elif 'missing_value' in var_attrs:
                 _fill_value = var_attrs['missing_value']
+                var[var == _fill_value] = numpy.nan
             elif 'fill_value' in var_attrs:
                 _fill_value = var_attrs['fill_value']
-            else:
-                logging.exception('egads - hdf_io.py - Hdf - _read_variable - KeyError, no missing value metadata '
-                                  + 'has been found for the variable ' + varname)
-                raise KeyError("ERROR: no missing value exists for Variable %s in %s" % (varname, self.filename))
-            if _fill_value is not None:
                 var[var == _fill_value] = numpy.nan
+            else:
+                logging.warning('egads - hdf_io.py - Hdf - _read_variable - KeyError, no missing value '
+                                + 'attribute has been found for the variable ' + varname + '. Missing value not '
+                                + 'replaced by NaN.')
         logging.debug('egads - hdf_io.py - Hdf - _read_variable - varname ' + str(varname) + ' -> data read OK')
         return var
 
@@ -1229,16 +1273,17 @@ def _read_variable(self, varname, input_range, read_as_float, replace_fill_value
             _fill_value = None
             if '_FillValue' in var_attrs:
                 _fill_value = var_attrs['_FillValue']
+                var[var == _fill_value] = numpy.nan
             elif 'missing_value' in var_attrs:
                 _fill_value = var_attrs['missing_value']
+                var[var == _fill_value] = numpy.nan
             elif 'fill_value' in var_attrs:
                 _fill_value = var_attrs['fill_value']
-            else:
-                logging.exception('egads - hdf_io.py - EgadsHdf - _read_variable - KeyError, no missing value '
-                                  'metadata has been found for the variable ' + varname)
-                raise KeyError("ERROR: no missing value exists for Variable %s in %s" % (varname, self.filename))
-            if _fill_value is not None:
                 var[var == _fill_value] = numpy.nan
+            else:
+                logging.warning('egads - hdf_io.py - EgadsHdf - _read_variable - KeyError, no missing value '
+                                + 'attribute has been found for the variable ' + varname + '. Missing value not '
+                                + 'replaced by NaN.')
 
         variable_metadata = egads.core.metadata.VariableMetadata(var_attrs, self.file_metadata)
         data = egads.EgadsData(var, variable_metadata=variable_metadata)

egads/input/netcdf_io.py

@@ -1396,8 +1396,9 @@ def _read_variable(self, varname, input_range, read_as_float, replace_fill_value
                     _fill_value = variable_attrs['missing_value']
                     value[value == _fill_value] = numpy.nan
                 else:
-                    logging.debug('egads - netcdf_io.py - EgadsNetCdf - _read_variable - varname ' + str(varname)
-                                  + ', no _FillValue or missing_value attribute found.')
+                    logging.warning('egads - netcdf_io.py - EgadsNetCdf - _read_variable - varname ' + str(varname)
+                                    + ', no _FillValue or missing_value attribute found. Fill value not replaced by '
+                                    + 'NaN.')
         variable_metadata = egads.core.metadata.VariableMetadata(variable_attrs, self.file_metadata)
         data = egads.EgadsData(value, variable_metadata=variable_metadata)
         logging.debug('egads - netcdf_io.py - EgadsNetCdf - _read_variable - varname ' + str(varname)

egads_lineage.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: egads-lineage
-Version: 1.2.3
+Version: 1.2.4
 Summary: EGADS: EUFAR General Airborne Data-processing Software
 Home-page: https://github.com/EUFAR/egads/tree/Lineage
 Author: EUFAR, Olivier Henry

egads_lineage.egg-info/SOURCES.txt

@@ -3,9 +3,8 @@ MANIFEST.in
 README.md
 setup.cfg
 setup.py
-Documentation/EGADS Lineage Algorithm Handbook - v1.2.2.pdf
-Documentation/EGADS Lineage Algorithm Handbook - v1.2.3.pdf
-Documentation/EGADS Lineage Documentation - v1.2.3.pdf
+Documentation/EGADS Lineage Algorithm Handbook - v1.2.4.pdf
+Documentation/EGADS Lineage Documentation - v1.2.4.pdf
 Documentation/changelog.txt
 Documentation/EGADS Algorithm Handbook - LATEX/EGADS_alg_doc.tex
 Documentation/EGADS Algorithm Handbook - LATEX/example.py

setup.py

@@ -30,7 +30,7 @@
 doclines = __doc__.split('\n')
 
 setup(name='egads-lineage',
-      version='1.2.3',
+      version='1.2.4',
       description=doclines[0],
       long_description='\n'.join(doclines[2:]),
       author='EUFAR, Olivier Henry',