Merge pull request #288 from FAIRmat-NFDI/apm-em-plugins-nxdefs-tracking

apm-em-plugins-nxdefs-tracking

dev-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile --extra=dev --extra=docs --output-file=dev-requirements.txt pyproject.toml
@@ -8,9 +8,9 @@ ase==3.22.1
     # via pynxtools (pyproject.toml)
 babel==2.14.0
     # via mkdocs-material
-build==1.0.3
+build==1.1.1
     # via pip-tools
-certifi==2023.11.17
+certifi==2024.2.2
     # via requests
 cfgv==3.4.0
     # via pre-commit
@@ -26,51 +26,49 @@ click-default-group==1.2.4
     # via pynxtools (pyproject.toml)
 colorama==0.4.6
     # via mkdocs-material
-contourpy==1.1.1
+contourpy==1.2.0
     # via matplotlib
-coverage[toml]==7.3.2
-    # via
-    #   coverage
-    #   pytest-cov
+coverage[toml]==7.4.4
+    # via pytest-cov
 cycler==0.12.1
     # via matplotlib
-distlib==0.3.7
+distlib==0.3.8
     # via virtualenv
-exceptiongroup==1.2.0
-    # via pytest
-filelock==3.13.1
+filelock==3.13.3
     # via virtualenv
-fonttools==4.45.1
+fonttools==4.50.0
     # via matplotlib
 ghp-import==2.1.0
     # via mkdocs
 h5py==3.10.0
     # via pynxtools (pyproject.toml)
-identify==2.5.33
+identify==2.5.35
     # via pre-commit
 idna==3.6
     # via requests
-importlib-metadata==6.8.0
+importlib-metadata==7.1.0
     # via pynxtools (pyproject.toml)
 iniconfig==2.0.0
     # via pytest
-jinja2==3.1.2
+jinja2==3.1.3
     # via
     #   mkdocs
     #   mkdocs-macros-plugin
     #   mkdocs-material
 kiwisolver==1.4.5
     # via matplotlib
-markdown==3.5.2
+lxml==5.1.0
+    # via pynxtools (pyproject.toml)
+markdown==3.6
     # via
     #   mkdocs
     #   mkdocs-material
     #   pymdown-extensions
-markupsafe==2.1.3
+markupsafe==2.1.5
     # via
     #   jinja2
     #   mkdocs
-matplotlib==3.7.4
+matplotlib==3.8.3
     # via ase
 mergedeep==1.3.4
     # via
@@ -83,19 +81,19 @@ mkdocs==1.5.3
     #   pynxtools (pyproject.toml)
 mkdocs-macros-plugin==1.0.5
     # via pynxtools (pyproject.toml)
-mkdocs-material==9.5.10
+mkdocs-material==9.5.15
     # via pynxtools (pyproject.toml)
 mkdocs-material-extensions==1.3.1
     # via
     #   mkdocs-material
     #   pynxtools (pyproject.toml)
-mypy==1.7.1
+mypy==1.9.0
     # via pynxtools (pyproject.toml)
 mypy-extensions==1.0.0
     # via mypy
 nodeenv==1.8.0
     # via pre-commit
-numpy==1.24.4
+numpy==1.26.4
     # via
     #   ase
     #   contourpy
@@ -105,7 +103,7 @@ numpy==1.24.4
     #   pynxtools (pyproject.toml)
     #   scipy
     #   xarray
-packaging==23.2
+packaging==24.0
     # via
     #   build
     #   matplotlib
@@ -114,48 +112,50 @@ packaging==23.2
     #   xarray
 paginate==0.5.6
     # via mkdocs-material
-pandas==2.0.3
+pandas==2.2.1
     # via
     #   pynxtools (pyproject.toml)
     #   xarray
 pathspec==0.12.1
     # via mkdocs
-pillow==10.0.1
+pillow==10.2.0
     # via matplotlib
-pip-tools==7.3.0
+pip-tools==7.4.1
     # via pynxtools (pyproject.toml)
-platformdirs==4.0.0
+platformdirs==4.2.0
     # via
     #   mkdocs
     #   virtualenv
-pluggy==1.3.0
+pluggy==1.4.0
     # via pytest
-pre-commit==3.5.0
+pre-commit==3.7.0
     # via pynxtools (pyproject.toml)
 pygments==2.17.2
     # via mkdocs-material
-pymdown-extensions==10.7
+pymdown-extensions==10.7.1
     # via mkdocs-material
-pyparsing==3.1.1
+pyparsing==3.1.2
     # via matplotlib
 pyproject-hooks==1.0.0
-    # via build
-pytest==7.4.3
+    # via
+    #   build
+    #   pip-tools
+pytest==8.1.1
     # via
     #   pynxtools (pyproject.toml)
     #   pytest-cov
     #   pytest-timeout
-pytest-cov==4.1.0
+pytest-cov==5.0.0
     # via pynxtools (pyproject.toml)
-pytest-timeout==2.2.0
+pytest-timeout==2.3.1
     # via pynxtools (pyproject.toml)
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via
     #   ghp-import
     #   matplotlib
     #   mkdocs-macros-plugin
     #   pandas
-pytz==2023.3.post1
+pytz==2024.1
     # via pandas
 pyyaml==6.0.1
     # via
@@ -173,45 +173,37 @@ requests==2.31.0
     # via mkdocs-material
 ruff==0.3.4
     # via pynxtools (pyproject.toml)
-scipy==1.9.1
+scipy==1.12.0
     # via ase
 six==1.16.0
     # via python-dateutil
-structlog==23.2.0
+structlog==24.1.0
     # via pynxtools (pyproject.toml)
 termcolor==2.4.0
     # via mkdocs-macros-plugin
-tomli==2.0.1
-    # via
-    #   build
-    #   coverage
-    #   mypy
-    #   pip-tools
-    #   pyproject-hooks
-    #   pytest
-types-pytz==2023.3.1.1
+types-pytz==2024.1.0.20240203
     # via pynxtools (pyproject.toml)
-types-pyyaml==6.0.12.12
+types-pyyaml==6.0.12.20240311
     # via pynxtools (pyproject.toml)
-types-requests==2.31.0.10
+types-requests==2.31.0.20240311
     # via pynxtools (pyproject.toml)
-typing-extensions==4.8.0
+typing-extensions==4.10.0
     # via mypy
-tzdata==2023.3
+tzdata==2024.1
     # via pandas
-urllib3==2.1.0
+urllib3==2.2.1
     # via
     #   requests
     #   types-requests
-virtualenv==20.25.0
+virtualenv==20.25.1
     # via pre-commit
 watchdog==4.0.0
     # via mkdocs
-wheel==0.42.0
+wheel==0.43.0
     # via pip-tools
-xarray==2023.1.0
+xarray==2024.2.0
     # via pynxtools (pyproject.toml)
-zipp==3.17.0
+zipp==3.18.1
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:

pynxtools/__init__.py

@@ -1,5 +1,3 @@
-"""init file"""
-
 #
 # Copyright The NOMAD Authors.
 #
@@ -17,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
 import os
 import re
 from datetime import datetime

pynxtools/dataconverter/helpers.py

@@ -20,17 +20,17 @@
 import json
 import logging
 import re
-import xml.etree.ElementTree as ET
 from datetime import datetime, timezone
 from typing import Any, Callable, List, Optional, Tuple, Union
 
 import h5py
+import lxml.etree as ET
 import numpy as np
 from ase.data import chemical_symbols
 
 from pynxtools import get_nexus_version, get_nexus_version_hash
 from pynxtools.nexus import nexus
-from pynxtools.nexus.nexus import NxdlAttributeError
+from pynxtools.nexus.nexus import NxdlAttributeNotFoundError
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -63,6 +63,9 @@ def get_all_defined_required_children_for_elem(xml_element):
     """Gets all possible inherited required children for a given NXDL element"""
     list_of_children_to_add = set()
     for child in xml_element:
+        tag = remove_namespace_from_tag(child.tag)
+        if tag not in ("group", "field", "attribute"):
+            continue
         child.set("nxdlbase_class", xml_element.get("nxdlbase_class"))
         if child.attrib and get_required_string(child) == "required":
             tag = remove_namespace_from_tag(child.tag)
@@ -572,7 +575,7 @@ def try_undocumented(data, nxdl_root: ET.Element):
             if units in data.undocumented:
                 data[get_required_string(elem)][units] = data.undocumented[units]
                 del data.undocumented[units]
-        except NxdlAttributeError:
+        except NxdlAttributeNotFoundError:
             pass
 
 
@@ -637,6 +640,8 @@ def validate_data_dict(template, data, nxdl_root: ET.Element):
 def remove_namespace_from_tag(tag):
     """Helper function to remove the namespace from an XML tag."""
 
+    if not isinstance(tag, str):
+        return ""
     return tag.split("}")[-1]
 
 

pynxtools/dataconverter/writer.py

@@ -28,8 +28,8 @@
 import numpy as np
 
 from pynxtools.dataconverter import helpers
-from pynxtools.nexus import nexus
 from pynxtools.dataconverter.exceptions import InvalidDictProvided
+from pynxtools.nexus import nexus
 
 logger = logging.getLogger(__name__)  # pylint: disable=C0103
 logger.setLevel(logging.INFO)
@@ -222,7 +222,7 @@ def __nxdl_to_attrs(self, path: str = "/") -> dict:
             elem = nexus.get_node_at_nxdl_path(
                 nxdl_path, elem=copy.deepcopy(self.nxdl_data)
             )
-        except nexus.NxdlAttributeError:
+        except nexus.NxdlAttributeNotFoundError:
             return None
 
         # Remove the name attribute as we only use it to name the HDF5 entry

pynxtools/nexus-version.txt

@@ -1 +1 @@
-v2020.10-1462-g7a476095
+v2020.10-1558-gda544c9b

pynxtools/nexus/nexus.py

@@ -4,11 +4,36 @@
 import logging
 import os
 import sys
+from functools import lru_cache
 
 import click
 import h5py
-
-from pynxtools.nexus.nxdl_utils import *  # pylint: disable=wildcard-import, unused-wildcard-import
+import lxml.etree as ET
+from pynxtools.definitions.dev_tools.utils.nxdl_utils import (
+    get_hdf_info_parent,
+    get_nxdl_child,
+    get_node_concept_path,
+    get_local_name_from_xml,
+    write_doc_string,
+    try_find_units,
+    check_attr_name_nxdl,
+    get_required_string,
+    get_best_child,
+    get_node_name,
+    add_base_classes,
+    walk_elist,
+    get_nx_class,
+    try_find_default,
+    other_attrs,
+    get_nexus_definitions_path,
+    NxdlAttributeNotFoundError,
+    get_node_at_nxdl_path,
+    get_enums,
+    get_inherited_nodes,
+    get_nx_classes,
+    get_nx_units,
+    get_nx_attribute_type,
+)
 
 
 def get_nxdl_entry(hdf_info):