From aeb798c1594e19d9533c02843f9a02fd6f473b88 Mon Sep 17 00:00:00 2001 From: Afonso Mukai Date: Mon, 8 Aug 2022 10:42:07 +0200 Subject: [PATCH] Restore definitions --- definitions/impatient-guide/conf.py | 163 +++--- definitions/manual/source/conf.py | 48 +- .../source/examples/epics/write_nexus_file.py | 132 +++-- .../examples/epics/write_nexus_file2.py | 54 +- .../source/examples/h5py/BasicReader.py | 10 +- .../source/examples/h5py/BasicWriter.py | 56 +- .../manual/source/examples/h5py/TestReader.py | 19 +- .../manual/source/examples/h5py/TestWriter.py | 33 +- .../source/examples/h5py/externalExample.py | 22 +- .../examples/h5py/reader_attributes_trail.py | 1 + .../manual/source/examples/h5py/writer_1_3.py | 14 +- .../manual/source/examples/h5py/writer_2_1.py | 32 +- .../manual/source/examples/simple3D.py | 20 +- .../manual/source/examples/verysimple.py | 86 +-- definitions/utils/build_preparation.py | 102 ++-- definitions/utils/create_release_notes.py | 101 ++-- definitions/utils/dev_create_release_notes.py | 12 +- definitions/utils/dev_nxdl2rst.py | 6 +- definitions/utils/dev_units2rst.py | 6 +- definitions/utils/local_utilities.py | 31 +- definitions/utils/nxdl2rst.py | 503 +++++++++--------- definitions/utils/nxdl_desc2rst.py | 335 ++++++------ definitions/utils/nxdl_summary.py | 92 ++-- definitions/utils/test_nxdl.py | 62 +-- definitions/utils/test_nxdl2rst.py | 68 ++- definitions/utils/test_suite.py | 11 +- definitions/utils/types2rst.py | 12 +- definitions/utils/units2rst.py | 84 +-- definitions/utils/update_copyright_date.py | 89 ++-- 29 files changed, 1059 insertions(+), 1145 deletions(-) diff --git a/definitions/impatient-guide/conf.py b/definitions/impatient-guide/conf.py index 9f3ddf6b4..165ad7d7a 100644 --- a/definitions/impatient-guide/conf.py +++ b/definitions/impatient-guide/conf.py @@ -16,95 +16,95 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +#sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' +#needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [] # Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] +templates_path = ['_templates'] # The suffix of source filenames. -source_suffix = ".rst" +source_suffix = '.rst' # The encoding of source files. -# source_encoding = 'utf-8-sig' +#source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = "index" +master_doc = 'index' # General information about the project. -project = u"NeXus for the Impatient" -copyright = u"2014-2016, http://nexusformat.org" +project = u'NeXus for the Impatient' +copyright = u'2014-2016, http://nexusformat.org' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = "2016" +version = '2016' # The full version, including alpha/beta/rc tags. -release = "2016" +release = '2016' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -# language = None +#language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -# today = '' +#today = '' # Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' +#today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ["_build"] +exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. -# default_role = None +#default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True +#add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -# add_module_names = True +#add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -# show_authors = False +#show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" +pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] +#modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "agogo" -html_theme = "sphinxdoc" +html_theme = 'agogo' +html_theme = 'sphinxdoc' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -# html_theme_options = {} +#html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] +#html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -html_title = "NeXus for the Impatient" +html_title = 'NeXus for the Impatient' # A shorter title for the navigation bar. Default is the same as html_title. html_short_title = project @@ -116,97 +116,100 @@ # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -html_favicon = "favicon.ico" +html_favicon = 'favicon.ico' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -html_last_updated_fmt = "%b %d, %Y" +html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -# html_use_smartypants = True +#html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -# html_sidebars = {} +#html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -# html_additional_pages = {} +#html_additional_pages = {} # If false, no module index is generated. -# html_domain_indices = True +#html_domain_indices = True # If false, no index is generated. html_use_index = False # If true, the index is split into individual pages for each letter. -# html_split_index = False +#html_split_index = False # If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True +#html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True +#html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True +#html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -# html_use_opensearch = '' +#html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None +#html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = "NXImpatient" +htmlhelp_basename = 'NXImpatient' # -- Options for LaTeX output -------------------------------------------------- latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - "papersize": "letterpaper", - # The font size ('10pt', '11pt' or '12pt'). - #'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # FIXME: roman page numbers in TOC, and no page numbers later - # http://osdir.com/ml/sphinx-dev/2011-03/msg00036.html - # BUT, latex does not recognize these two lines when in the preamble - "preamble": """% +# The paper size ('letterpaper' or 'a4paper'). +'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +# FIXME: roman page numbers in TOC, and no page numbers later +# http://osdir.com/ml/sphinx-dev/2011-03/msg00036.html +# BUT, latex does not recognize these two lines when in the preamble +'preamble': '''% \pagestyle{plain} \pagenumbering{arabic} -""", +''', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ("index", "NXImpatient.tex", project, u"nexusformat.org", "howto"), + ('index', 'NXImpatient.tex', project, + u'nexusformat.org', 'howto'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -latex_logo = "nexuslogo.png" +latex_logo = 'nexuslogo.png' # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -# latex_use_parts = False +#latex_use_parts = False # If true, show page references after internal links. -# latex_show_pagerefs = False +#latex_show_pagerefs = False # If true, show URL addresses after external links. -# latex_show_urls = False +#latex_show_urls = False # Documents to append as an appendix to all manuals. -# latex_appendices = [] +#latex_appendices = [] # If false, no module index is generated. latex_domain_indices = False @@ -216,10 +219,13 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [("index", "nximpatient", project, [u"nexusformat.org"], 1)] +man_pages = [ + ('index', 'nximpatient', project, + [u'nexusformat.org'], 1) +] # If true, show URL addresses after external links. -# man_show_urls = False +#man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ @@ -228,65 +234,60 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ( - "index", - "NXImpatient", - project, - u"nexusformat.org", - "NXImpatient", - "Brief overview of the NeXus data format.", - "Miscellaneous", - ), + ('index', 'NXImpatient', project, + u'nexusformat.org', 'NXImpatient', + 'Brief overview of the NeXus data format.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. -# texinfo_appendices = [] +#texinfo_appendices = [] # If false, no module index is generated. -# texinfo_domain_indices = True +#texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -# texinfo_show_urls = 'footnote' +#texinfo_show_urls = 'footnote' # -- Options for Epub output --------------------------------------------------- # Bibliographic Dublin Core info. epub_title = project -epub_author = u"nexusformat.org" -epub_publisher = u"http://nexusformat.org" +epub_author = u'nexusformat.org' +epub_publisher = u'http://nexusformat.org' epub_copyright = copyright # The language of the text. It defaults to the language option # or en if the language is not set. -# epub_language = '' +#epub_language = '' # The scheme of the identifier. Typical schemes are ISBN or URL. -# epub_scheme = '' +#epub_scheme = '' # The unique identifier of the text. This can be a ISBN number # or the project homepage. -# epub_identifier = '' +#epub_identifier = '' # A unique identification for the text. -# epub_uid = '' +#epub_uid = '' # A tuple containing the cover image and cover page html template filenames. -# epub_cover = () +#epub_cover = () # HTML files that should be inserted before the pages created by sphinx. # The format is a list of tuples containing the path and title. -# epub_pre_files = [] +#epub_pre_files = [] # HTML files shat should be inserted after the pages created by sphinx. # The format is a list of tuples containing the path and title. -# epub_post_files = [] +#epub_post_files = [] # A list of files that should not be packed into the epub file. -# epub_exclude_files = [] +#epub_exclude_files = [] # The depth of the table of contents in toc.ncx. -# epub_tocdepth = 3 +#epub_tocdepth = 3 # Allow duplicate toc entries. -# epub_tocdup = True +#epub_tocdup = True diff --git a/definitions/manual/source/conf.py b/definitions/manual/source/conf.py index 1188a3615..0160d58f2 100644 --- a/definitions/manual/source/conf.py +++ b/definitions/manual/source/conf.py @@ -19,37 +19,37 @@ # -- Project information ----------------------------------------------------- -project = "nexus" -author = "NIAC, https://www.nexusformat.org" -copyright = u"1996-{}, {}".format(datetime.datetime.now().year, author) -description = u"NeXus: A Common Data Format for Neutron, X-ray, and Muon Science" +project = 'nexus' +author = 'NIAC, https://www.nexusformat.org' +copyright = u'1996-{}, {}'.format(datetime.datetime.now().year, author) +description = u'NeXus: A Common Data Format for Neutron, X-ray, and Muon Science' # The full version, including alpha/beta/rc tags -version = u"unknown NXDL version" -release = u"unknown NXDL release" -nxdl_version = open("../../NXDL_VERSION").read().strip() +version = u'unknown NXDL version' +release = u'unknown NXDL release' +nxdl_version = open('../../NXDL_VERSION').read().strip() if nxdl_version is not None: - version = nxdl_version.split(".")[0] + version = nxdl_version.split('.')[0] release = nxdl_version # -- General configuration --------------------------------------------------- # https://github.com/nexusformat/definitions/issues/659#issuecomment-577438319 -needs_sphinx = "2.3" +needs_sphinx = '2.3' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - "sphinx.ext.mathjax", - "sphinx.ext.ifconfig", - "sphinx.ext.viewcode", - "sphinx.ext.githubpages", + 'sphinx.ext.mathjax', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', ] # Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] +templates_path = ['_templates'] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -63,27 +63,27 @@ # a list of builtin themes. # # html_theme = 'alabaster' -html_theme = "sphinxdoc" +html_theme = 'sphinxdoc' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +html_static_path = ['_static'] html_sidebars = { - "**": [ - "localtoc.html", - "relations.html", - "sourcelink.html", - "searchbox.html", - "google_search.html", + '**': [ + 'localtoc.html', + 'relations.html', + 'sourcelink.html', + 'searchbox.html', + 'google_search.html' ], } # Output file base name for HTML help builder. -htmlhelp_basename = "NeXusManualdoc" +htmlhelp_basename = 'NeXusManualdoc' # -- Options for Latex output ------------------------------------------------- latex_elements = { - "maxlistdepth": 7, # some application definitions are deeply nested + 'maxlistdepth':7, # some application definitions are deeply nested } diff --git a/definitions/manual/source/examples/epics/write_nexus_file.py b/definitions/manual/source/examples/epics/write_nexus_file.py index a6ef7e108..1dd61bafb 100755 --- a/definitions/manual/source/examples/epics/write_nexus_file.py +++ b/definitions/manual/source/examples/epics/write_nexus_file.py @@ -2,83 +2,81 @@ import h5py import datetime - def write_nexus_file(fname, image, md={}): - """ - write the image to a NeXus HDF5 data file - - Parameters - ---------- - fname : str - name of the file (relative or absolute) to be written - image : numpy array - the image data - md : dictionary - key: value where value is something that can be written by h5py - (such as str, int, float, numpy array, ...) - """ - nexus = h5py.File(fname, "w") - nexus.attrs["filename"] = fname - nexus.attrs["file_time"] = str(datetime.datetime.now()) - nexus.attrs["creator"] = "write_nexus_file()" - nexus.attrs["H5PY_VERSION"] = h5py.__version__ + """ + write the image to a NeXus HDF5 data file - # /entry - nxentry = nexus.create_group("entry") - nxentry.attrs["NX_class"] = "NXentry" - nexus.attrs["default"] = nxentry.name + Parameters + ---------- + fname : str + name of the file (relative or absolute) to be written + image : numpy array + the image data + md : dictionary + key: value where value is something that can be written by h5py + (such as str, int, float, numpy array, ...) + """ + nexus = h5py.File(fname, "w") + nexus.attrs["filename"] = fname + nexus.attrs["file_time"] = str(datetime.datetime.now()) + nexus.attrs["creator"] = "write_nexus_file()" + nexus.attrs["H5PY_VERSION"] = h5py.__version__ - # /entry/instrument - nxinstrument = nxentry.create_group("instrument") - nxinstrument.attrs["NX_class"] = "NXinstrument" + # /entry + nxentry = nexus.create_group("entry") + nxentry.attrs["NX_class"] = "NXentry" + nexus.attrs["default"] = nxentry.name - # /entry/instrument/detector - nxdetector = nxinstrument.create_group("detector") - nxdetector.attrs["NX_class"] = "NXdetector" + # /entry/instrument + nxinstrument = nxentry.create_group("instrument") + nxinstrument.attrs["NX_class"] = "NXinstrument" - # /entry/instrument/detector/image - ds = nxdetector.create_dataset("image", data=image, compression="gzip") - ds.attrs["units"] = "counts" - ds.attrs["target"] = "/entry/instrument/detector/image" + # /entry/instrument/detector + nxdetector = nxinstrument.create_group("detector") + nxdetector.attrs["NX_class"] = "NXdetector" - # /entry/data - nxdata = nxentry.create_group("data") - nxdata.attrs["NX_class"] = "NXdata" - nxentry.attrs["default"] = nxdata.name + # /entry/instrument/detector/image + ds = nxdetector.create_dataset("image", data=image, compression="gzip") + ds.attrs["units"] = "counts" + ds.attrs["target"] = "/entry/instrument/detector/image" - # /entry/data/data --> /entry/instrument/detector/image - nxdata["data"] = nexus["/entry/instrument/detector/image"] - nxdata.attrs["signal"] = "data" + # /entry/data + nxdata = nxentry.create_group("data") + nxdata.attrs["NX_class"] = "NXdata" + nxentry.attrs["default"] = nxdata.name - if len(md) > 0: - # /entry/instrument/metadata (optional, for metadata) - metadata = nxinstrument.create_group("metadata") - metadata.attrs["NX_class"] = "NXcollection" - for k, v in md.items(): - try: - metadata.create_dataset(k, data=v) - except Exception: - metadata.create_dataset(k, data=str(v)) + # /entry/data/data --> /entry/instrument/detector/image + nxdata["data"] = nexus["/entry/instrument/detector/image"] + nxdata.attrs["signal"] = "data" - nexus.close() + if len(md) > 0: + # /entry/instrument/metadata (optional, for metadata) + metadata = nxinstrument.create_group("metadata") + metadata.attrs["NX_class"] = "NXcollection" + for k, v in md.items(): + try: + metadata.create_dataset(k, data=v) + except Exception: + metadata.create_dataset(k, data=str(v)) + nexus.close() + if __name__ == "__main__": - """demonstrate how to use this code""" - import epics - - prefix = "13SIM1:" - img = epics.caget(prefix + "image1:ArrayData") - size_x = epics.caget(prefix + "cam1:ArraySizeX_RBV") - size_y = epics.caget(prefix + "cam1:ArraySizeY_RBV") - # edit the full image for just the binned data - img = img[: size_x * size_y].reshape((size_x, size_y)) + """demonstrate how to use this code""" + import epics + prefix = "13SIM1:" + img = epics.caget(prefix+"image1:ArrayData") + size_x = epics.caget(prefix+"cam1:ArraySizeX_RBV") + size_y = epics.caget(prefix+"cam1:ArraySizeY_RBV") + # edit the full image for just the binned data + img = img[:size_x*size_y].reshape((size_x, size_y)) - extra_information = dict( - unique_id=epics.caget(prefix + "image1:UniqueId_RBV"), - size_x=size_x, - size_y=size_y, - detector_state=epics.caget(prefix + "cam1:DetectorState_RBV"), - bitcoin_value="15000", - ) - write_nexus_file("example.h5", img, md=extra_information) + extra_information = dict( + unique_id = epics.caget(prefix+"image1:UniqueId_RBV"), + size_x = size_x, + size_y = size_y, + detector_state = epics.caget(prefix+"cam1:DetectorState_RBV"), + bitcoin_value="15000", + ) + write_nexus_file("example.h5", img, md=extra_information) diff --git a/definitions/manual/source/examples/epics/write_nexus_file2.py b/definitions/manual/source/examples/epics/write_nexus_file2.py index ac883c809..8c63a068d 100755 --- a/definitions/manual/source/examples/epics/write_nexus_file2.py +++ b/definitions/manual/source/examples/epics/write_nexus_file2.py @@ -17,39 +17,37 @@ def write_nexus_file(fname, image, md={}): (such as str, int, float, numpy array, ...) """ nx = NXroot() - nx["/entry"] = NXentry(NXinstrument(NXdetector())) - nx["entry/instrument/detector/image"] = NXfield( - image, units="counts", compression="gzip" - ) - nx["entry/data"] = NXdata() - nx["entry/data"].makelink(nx["entry/instrument/detector/image"]) - nx["entry/data"].nxsignal = nx["entry/data/image"] + nx['/entry'] = NXentry(NXinstrument(NXdetector())) + nx['entry/instrument/detector/image'] = NXfield(image, units='counts', + compression='gzip') + nx['entry/data'] = NXdata() + nx['entry/data'].makelink(nx['entry/instrument/detector/image']) + nx['entry/data'].nxsignal = nx['entry/data/image'] if len(md) > 0: # /entry/instrument/metadata (optional, for metadata) - metadata = nx["/entry/instrument/metadata"] = NXcollection() + metadata = nx['/entry/instrument/metadata'] = NXcollection() for k, v in md.items(): metadata[k] = v - nx.save(fname, "w") - + nx.save(fname, 'w') + if __name__ == "__main__": - """demonstrate how to use this code""" - import epics - - prefix = "13SIM1:" - img = epics.caget(prefix + "image1:ArrayData") - size_x = epics.caget(prefix + "cam1:ArraySizeX_RBV") - size_y = epics.caget(prefix + "cam1:ArraySizeY_RBV") - # edit the full image for just the binned data - img = img[: size_x * size_y].reshape((size_x, size_y)) - - extra_information = dict( - unique_id=epics.caget(prefix + "image1:UniqueId_RBV"), - size_x=size_x, - size_y=size_y, - detector_state=epics.caget(prefix + "cam1:DetectorState_RBV"), - bitcoin_value="15000", - ) - write_nexus_file("example.h5", img, md=extra_information) + """demonstrate how to use this code""" + import epics + prefix = "13SIM1:" + img = epics.caget(prefix+"image1:ArrayData") + size_x = epics.caget(prefix+"cam1:ArraySizeX_RBV") + size_y = epics.caget(prefix+"cam1:ArraySizeY_RBV") + # edit the full image for just the binned data + img = img[:size_x*size_y].reshape((size_x, size_y)) + + extra_information = dict( + unique_id = epics.caget(prefix+"image1:UniqueId_RBV"), + size_x = size_x, + size_y = size_y, + detector_state = epics.caget(prefix+"cam1:DetectorState_RBV"), + bitcoin_value="15000", + ) + write_nexus_file("example.h5", img, md=extra_information) diff --git a/definitions/manual/source/examples/h5py/BasicReader.py b/definitions/manual/source/examples/h5py/BasicReader.py index a3e2875ba..3a499c5c7 100755 --- a/definitions/manual/source/examples/h5py/BasicReader.py +++ b/definitions/manual/source/examples/h5py/BasicReader.py @@ -1,14 +1,14 @@ #!/usr/bin/env python -"""Reads NeXus HDF5 files using h5py and prints the contents""" +'''Reads NeXus HDF5 files using h5py and prints the contents''' -import h5py # HDF5 support +import h5py # HDF5 support fileName = "prj_test.nexus.hdf5" -f = h5py.File(fileName, "r") +f = h5py.File(fileName, "r") for item in f.attrs.keys(): print(item + ":", f.attrs[item]) -mr = f["/entry/mr_scan/mr"] -i00 = f["/entry/mr_scan/I00"] +mr = f['/entry/mr_scan/mr'] +i00 = f['/entry/mr_scan/I00'] print("%s\t%s\t%s" % ("#", "mr", "I00")) for i in range(len(mr)): print("%d\t%g\t%d" % (i, mr[i], i00[i])) diff --git a/definitions/manual/source/examples/h5py/BasicWriter.py b/definitions/manual/source/examples/h5py/BasicWriter.py index f71925d96..63ccffde1 100755 --- a/definitions/manual/source/examples/h5py/BasicWriter.py +++ b/definitions/manual/source/examples/h5py/BasicWriter.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -"""Writes a NeXus HDF5 file using h5py and numpy""" +'''Writes a NeXus HDF5 file using h5py and numpy''' -import h5py # HDF5 support +import h5py # HDF5 support import numpy print("Write a NeXus HDF5 file") @@ -11,46 +11,44 @@ # load data from two column format data = numpy.loadtxt("input.dat").T mr_arr = data[0] -i00_arr = numpy.asarray(data[1], "int32") +i00_arr = numpy.asarray(data[1],'int32') # create the HDF5 NeXus file f = h5py.File(fileName, "w") # point to the default data to be plotted -f.attrs["default"] = "entry" +f.attrs['default'] = 'entry' # give the HDF5 root some more attributes -f.attrs["file_name"] = fileName -f.attrs["file_time"] = timestamp -f.attrs["instrument"] = "APS USAXS at 32ID-B" -f.attrs["creator"] = "BasicWriter.py" -f.attrs["NeXus_version"] = "4.3.0" -f.attrs["HDF5_Version"] = h5py.version.hdf5_version -f.attrs["h5py_version"] = h5py.version.version +f.attrs['file_name'] = fileName +f.attrs['file_time'] = timestamp +f.attrs['instrument'] = 'APS USAXS at 32ID-B' +f.attrs['creator'] = 'BasicWriter.py' +f.attrs['NeXus_version'] = '4.3.0' +f.attrs['HDF5_Version'] = h5py.version.hdf5_version +f.attrs['h5py_version'] = h5py.version.version # create the NXentry group -nxentry = f.create_group("entry") -nxentry.attrs["NX_class"] = "NXentry" -nxentry.attrs["default"] = "mr_scan" -nxentry.create_dataset("title", data="1-D scan of I00 v. mr") +nxentry = f.create_group('entry') +nxentry.attrs['NX_class'] = 'NXentry' +nxentry.attrs['default'] = 'mr_scan' +nxentry.create_dataset('title', data='1-D scan of I00 v. mr') # create the NXentry group -nxdata = nxentry.create_group("mr_scan") -nxdata.attrs["NX_class"] = "NXdata" -nxdata.attrs["signal"] = "I00" # Y axis of default plot -nxdata.attrs["axes"] = "mr" # X axis of default plot -nxdata.attrs["mr_indices"] = [ - 0, -] # use "mr" as the first dimension of I00 +nxdata = nxentry.create_group('mr_scan') +nxdata.attrs['NX_class'] = 'NXdata' +nxdata.attrs['signal'] = 'I00' # Y axis of default plot +nxdata.attrs['axes'] = 'mr' # X axis of default plot +nxdata.attrs['mr_indices'] = [0,] # use "mr" as the first dimension of I00 # X axis data -ds = nxdata.create_dataset("mr", data=mr_arr) -ds.attrs["units"] = "degrees" -ds.attrs["long_name"] = "USAXS mr (degrees)" # suggested X axis plot label +ds = nxdata.create_dataset('mr', data=mr_arr) +ds.attrs['units'] = 'degrees' +ds.attrs['long_name'] = 'USAXS mr (degrees)' # suggested X axis plot label # Y axis data -ds = nxdata.create_dataset("I00", data=i00_arr) -ds.attrs["units"] = "counts" -ds.attrs["long_name"] = "USAXS I00 (counts)" # suggested Y axis plot label +ds = nxdata.create_dataset('I00', data=i00_arr) +ds.attrs['units'] = 'counts' +ds.attrs['long_name'] = 'USAXS I00 (counts)' # suggested Y axis plot label -f.close() # be CERTAIN to close the file +f.close() # be CERTAIN to close the file print("wrote file:", fileName) diff --git a/definitions/manual/source/examples/h5py/TestReader.py b/definitions/manual/source/examples/h5py/TestReader.py index 538622066..e654ec54c 100755 --- a/definitions/manual/source/examples/h5py/TestReader.py +++ b/definitions/manual/source/examples/h5py/TestReader.py @@ -1,6 +1,7 @@ -"""Reads NeXus HDF5 files using h5py and prints the contents""" -import h5py # HDF5 support +'''Reads NeXus HDF5 files using h5py and prints the contents''' + +import h5py # HDF5 support testFiles = ("prj_test.nexus.hdf5",) @@ -15,18 +16,18 @@ def print_attr(parent, label): def print_child(item, label): - print("#" + "-" * 40) + print("#" + "-"*40) print("%s %s" % (label, type(item))) print_attr(item, label) - if repr(type(item)) == GROUP_TYPE_MATCH: + if (repr(type(item)) == GROUP_TYPE_MATCH): base = label if "NX_class" in item.attrs: base += ":" + item.attrs["NX_class"] for k in item.keys(): key = "%s/%s" % (base, k) print_child(item[k], key) - if repr(type(item)) == DATASET_TYPE_MATCH: - # print label, item.value + if (repr(type(item)) == DATASET_TYPE_MATCH): + #print label, item.value print(label) print("shape:", item.shape) print("size:", len(item.shape)) @@ -47,8 +48,8 @@ def process(fileName): return True -if __name__ == "__main__": +if __name__ == '__main__': for fileName in testFiles: - print("#" + "=" * 60) - if not process(fileName): + print("#" + "="*60) + if not process( fileName ): print("Could not open:", fileName) diff --git a/definitions/manual/source/examples/h5py/TestWriter.py b/definitions/manual/source/examples/h5py/TestWriter.py index cab40f985..64a358411 100755 --- a/definitions/manual/source/examples/h5py/TestWriter.py +++ b/definitions/manual/source/examples/h5py/TestWriter.py @@ -1,6 +1,6 @@ -"""Writes a NeXus HDF5 file using h5py""" +'''Writes a NeXus HDF5 file using h5py''' -import h5py # HDF5 support +import h5py # HDF5 support import time @@ -39,17 +39,17 @@ """ -if __name__ == "__main__": +if __name__ == '__main__': print("Write a NeXus HDF5 file") fileName = "prj_test.nexus.hdf5" tzsecs = abs(time.timezone) if time.timezone < 0: - tzhhmm = "+" # reverse logic, it seems + tzhhmm = "+" # reverse logic, it seems else: tzhhmm = "-" if time.daylight: tzsecs -= 3600 - tzhhmm += "%02d%02d" % (tzsecs / 3600, (tzsecs % 3600) / 60) + tzhhmm += "%02d%02d" % (tzsecs / 3600, (tzsecs % 3600)/60) timestamp = time.strftime("%Y-%m-%dT%H:%M:%S") + tzhhmm # prepare the data @@ -69,17 +69,17 @@ f.attrs["h5py_version"] = h5py.version.version f.attrs["file_time"] = timestamp f.attrs["file_update_time"] = timestamp - f.attrs["default"] = "entry" # identify default NXentry group + f.attrs["default"] = "entry" # identify default NXentry group nxentry = f.create_group("entry") - nxentry.attrs["NX_class"] = "NXentry" # identify NeXus base class - nxentry.attrs["default"] = "mr_scan" # identify default NXdata group + nxentry.attrs["NX_class"] = "NXentry" # identify NeXus base class + nxentry.attrs["default"] = "mr_scan" # identify default NXdata group # store the scan data nxdata = nxentry.create_group("mr_scan") - nxdata.attrs["NX_class"] = "NXdata" # identify NeXus base class - nxdata.attrs["signal"] = "I00" # identify default data to plot - nxdata.attrs["axes"] = "mr" # identify default dimension scale to plot + nxdata.attrs["NX_class"] = "NXdata" # identify NeXus base class + nxdata.attrs["signal"] = "I00" # identify default data to plot + nxdata.attrs["axes"] = "mr" # identify default dimension scale to plot mr = nxdata.create_dataset("mr", data=data["mr"]) mr.attrs["units"] = "degrees" @@ -88,13 +88,14 @@ i00.attrs["units"] = "counts" # fill in some optional metadata - nxentry.create_dataset("title", data="APS USAXS instrument MR (alignment) scan") + nxentry.create_dataset("title", + data="APS USAXS instrument MR (alignment) scan") nxentry.create_dataset("start_time", data="2010-04-25T10:20:56-0500") nxentry.create_dataset("end_time", data="2010-04-25T10:21:16-0500") - nxentry.create_dataset("experiment_identifier", data="spec file 04_25.dat, scan #8") - nxentry.create_dataset( - "experiment_description", data="alignment scan of the USAXS collimating optics" - ) + nxentry.create_dataset("experiment_identifier", + data="spec file 04_25.dat, scan #8") + nxentry.create_dataset("experiment_description", + data="alignment scan of the USAXS collimating optics") # be CERTAIN to close the file f.close() diff --git a/definitions/manual/source/examples/h5py/externalExample.py b/definitions/manual/source/examples/h5py/externalExample.py index fad5078c6..809cb1e1f 100755 --- a/definitions/manual/source/examples/h5py/externalExample.py +++ b/definitions/manual/source/examples/h5py/externalExample.py @@ -1,9 +1,9 @@ #!/usr/bin/env python -""" +''' Writes a NeXus HDF5 file using h5py with links to data in other HDF5 files. This example is based on ``writer_2_1``. -""" +''' import h5py import numpy @@ -12,21 +12,21 @@ FILE_HDF5_ANGLES = u"external_angles.hdf5" FILE_HDF5_COUNTS = u"external_counts.hdf5" -# --------------------------- +#--------------------------- # get some data buffer = numpy.loadtxt("input.dat").T -tthData = buffer[0] # float[] -countsData = numpy.asarray(buffer[1], "int32") # int[] +tthData = buffer[0] # float[] +countsData = numpy.asarray(buffer[1],'int32') # int[] # put the angle data in an external (non-NeXus) HDF5 data file f = h5py.File(FILE_HDF5_ANGLES, "w") ds = f.create_dataset(u"angles", data=tthData) ds.attrs[u"units"] = u"degrees" -f.close() # be CERTAIN to close the file +f.close() # be CERTAIN to close the file -# put the detector counts in an external HDF5 data file +# put the detector counts in an external HDF5 data file # with *incomplete* NeXus structure (no NXdata group) f = h5py.File(FILE_HDF5_COUNTS, "w") nxentry = f.create_group(u"entry") @@ -46,13 +46,13 @@ f = h5py.File(FILE_HDF5_MASTER, "w") f.attrs[u"default"] = u"entry" nxentry = f.create_group(u"entry") -nxentry.attrs[u"NX_class"] = u"NXentry" +nxentry.attrs[u"NX_class"] =u"NXentry" nxentry.attrs[u"default"] = u"data" nxdata = nxentry.create_group(u"data") nxdata.attrs[u"NX_class"] = u"NXdata" # link in the signal data -local_addr = "/entry/data/counts" +local_addr = '/entry/data/counts' external_addr = u"/entry/instrument/detector/counts" f[local_addr] = h5py.ExternalLink(FILE_HDF5_COUNTS, external_addr) nxdata.attrs[u"signal"] = u"counts" @@ -61,9 +61,7 @@ local_addr = u"/entry/data/two_theta" f[local_addr] = h5py.ExternalLink(FILE_HDF5_ANGLES, u"/angles") nxdata.attrs[u"axes"] = u"two_theta" -nxdata.attrs[u"two_theta_indices"] = [ - 0, -] +nxdata.attrs[u"two_theta_indices"] = [0,] local_addr = u"/entry/instrument" f[local_addr] = h5py.ExternalLink(FILE_HDF5_COUNTS, u"/entry/instrument") diff --git a/definitions/manual/source/examples/h5py/reader_attributes_trail.py b/definitions/manual/source/examples/h5py/reader_attributes_trail.py index 908ca7bed..acb0127ea 100644 --- a/definitions/manual/source/examples/h5py/reader_attributes_trail.py +++ b/definitions/manual/source/examples/h5py/reader_attributes_trail.py @@ -1,3 +1,4 @@ + import h5py with h5py.File("prj_test.nexus.hdf5", "r") as nx: diff --git a/definitions/manual/source/examples/h5py/writer_1_3.py b/definitions/manual/source/examples/h5py/writer_1_3.py index 3126f932f..dd4e15d67 100755 --- a/definitions/manual/source/examples/h5py/writer_1_3.py +++ b/definitions/manual/source/examples/h5py/writer_1_3.py @@ -1,18 +1,18 @@ #!/usr/bin/env python -""" +''' Writes the simplest NeXus HDF5 file using h5py Uses method accepted at 2014NIAC according to the example from Figure 1.3 in the Introduction chapter -""" +''' import h5py import numpy buffer = numpy.loadtxt("input.dat").T -tthData = buffer[0] # float[] -countsData = numpy.asarray(buffer[1], "int32") # int[] +tthData = buffer[0] # float[] +countsData = numpy.asarray(buffer[1],'int32') # int[] f = h5py.File("writer_1_3.hdf5", "w") # create the HDF5 NeXus file # since this is a simple example, no attributes are used at this point @@ -24,9 +24,7 @@ nxdata.attrs["NX_class"] = u"NXdata" nxdata.attrs[u"signal"] = u"counts" nxdata.attrs[u"axes"] = u"two_theta" -nxdata.attrs[u"two_theta_indices"] = [ - 0, -] +nxdata.attrs[u"two_theta_indices"] = [0,] tth = nxdata.create_dataset(u"two_theta", data=tthData) tth.attrs[u"units"] = u"degrees" @@ -34,4 +32,4 @@ counts = nxdata.create_dataset(u"counts", data=countsData) counts.attrs[u"units"] = u"counts" -f.close() # be CERTAIN to close the file +f.close() # be CERTAIN to close the file diff --git a/definitions/manual/source/examples/h5py/writer_2_1.py b/definitions/manual/source/examples/h5py/writer_2_1.py index 64be0142f..66e76b371 100755 --- a/definitions/manual/source/examples/h5py/writer_2_1.py +++ b/definitions/manual/source/examples/h5py/writer_2_1.py @@ -1,15 +1,15 @@ #!/usr/bin/env python -""" +''' Writes a simple NeXus HDF5 file using h5py with links according to the example from Figure 2.1 in the Design chapter -""" +''' import h5py import numpy buffer = numpy.loadtxt("input.dat").T -tthData = buffer[0] # float[] -countsData = numpy.asarray(buffer[1], "int32") # int[] +tthData = buffer[0] # float[] +countsData = numpy.asarray(buffer[1],'int32') # int[] f = h5py.File("writer_2_1.hdf5", "w") # create the HDF5 NeXus file f.attrs[u"default"] = u"entry" @@ -35,20 +35,18 @@ nxdata.attrs[u"NX_class"] = u"NXdata" nxdata.attrs[u"signal"] = u"counts" nxdata.attrs[u"axes"] = u"two_theta" -nxdata.attrs[u"two_theta_indices"] = [ - 0, -] - -source_addr = u"/entry/instrument/detector/two_theta" # existing data -target_addr = u"two_theta" # new location -ds_tth.attrs[u"target"] = source_addr # a NeXus API convention for links -nxdata[target_addr] = f[source_addr] # hard link +nxdata.attrs[u"two_theta_indices"] = [0,] + +source_addr = u"/entry/instrument/detector/two_theta" # existing data +target_addr = u"two_theta" # new location +ds_tth.attrs[u"target"] = source_addr # a NeXus API convention for links +nxdata[target_addr] = f[source_addr] # hard link # nxdata._id.link(source_addr, target_addr, h5py.h5g.LINK_HARD) -source_addr = u"/entry/instrument/detector/counts" # existing data -target_addr = u"counts" # new location -ds_counts.attrs[u"target"] = source_addr # a NeXus API convention for links -nxdata[target_addr] = f[source_addr] # hard link +source_addr = u"/entry/instrument/detector/counts" # existing data +target_addr = u"counts" # new location +ds_counts.attrs[u"target"] = source_addr # a NeXus API convention for links +nxdata[target_addr] = f[source_addr] # hard link # nxdata._id.link(source_addr, target_addr, h5py.h5g.LINK_HARD) -f.close() # be CERTAIN to close the file +f.close() # be CERTAIN to close the file diff --git a/definitions/manual/source/examples/simple3D.py b/definitions/manual/source/examples/simple3D.py index 0f3381cd1..3f5554684 100755 --- a/definitions/manual/source/examples/simple3D.py +++ b/definitions/manual/source/examples/simple3D.py @@ -4,30 +4,30 @@ import nxs import numpy -a = numpy.zeros((2, 3, 4), dtype=numpy.int) +a = numpy.zeros((2,3,4),dtype=numpy.int) val = 0 for i in range(2): for j in range(3): for k in range(4): - a[i, j, k] = val + a[i,j,k] = val val = val + 1 nf = nxs.open("simple3D.h5", "w5") -nf.makegroup("entry", "NXentry") -nf.opengroup("entry", "NXentry") +nf.makegroup("entry","NXentry") +nf.opengroup("entry","NXentry") -nf.makegroup("data", "NXdata") -nf.opengroup("data", "NXdata") -nf.putattr("signal", "test") +nf.makegroup("data","NXdata") +nf.opengroup("data","NXdata") +nf.putattr("signal","test") -nf.makedata("test", "int32", [2, 3, 4]) +nf.makedata("test",'int32',[2,3,4]) nf.opendata("test") nf.putdata(a) nf.closedata() -nf.closegroup() # NXdata -nf.closegroup() # NXentry +nf.closegroup() # NXdata +nf.closegroup() # NXentry nf.close() diff --git a/definitions/manual/source/examples/verysimple.py b/definitions/manual/source/examples/verysimple.py index 4a086c351..1288d4337 100755 --- a/definitions/manual/source/examples/verysimple.py +++ b/definitions/manual/source/examples/verysimple.py @@ -1,64 +1,34 @@ #!/usr/bin/env python -"""uses h5py to build the verysimple.nx5 data file""" +'''uses h5py to build the verysimple.nx5 data file''' import h5py -angle = [ - 18.9094, - 18.9096, - 18.9098, - 18.91, - 18.9102, - 18.9104, - 18.9106, - 18.9108, - 18.911, - 18.9112, - 18.9114, - 18.9116, - 18.9118, - 18.912, - 18.9122, -] -diode = [ - 1193, - 4474, - 53220, - 274310, - 515430, - 827880, - 1227100, - 1434640, - 1330280, - 1037070, - 598720, - 316460, - 56677, - 1000, - 1000, -] - -f = h5py.File("verysimple.nx5", "w") -f.attrs["default"] = "entry" - -nxentry = f.create_group("entry") -nxentry.attrs["NX_class"] = "NXentry" -nxentry.attrs["default"] = "data" - -nxdata = nxentry.create_group("data") -nxdata.attrs["NX_class"] = "NXdata" -nxdata.attrs["signal"] = "counts" -nxdata.attrs["axes"] = "two_theta" -nxdata.attrs["two_theta_indices"] = [ - 0, -] - -tth = nxdata.create_dataset("two_theta", data=angle) -tth.attrs["units"] = "degrees" -tth.attrs["long_name"] = "two_theta (degrees)" - -counts = nxdata.create_dataset("counts", data=diode) -counts.attrs["units"] = "counts" -counts.attrs["long_name"] = "photodiode counts" +angle = [18.9094, 18.9096, 18.9098, 18.91, 18.9102, + 18.9104, 18.9106, 18.9108, 18.911, 18.9112, + 18.9114, 18.9116, 18.9118, 18.912, 18.9122] +diode = [1193, 4474, 53220, 274310, 515430, 827880, + 1227100, 1434640, 1330280, 1037070, 598720, + 316460, 56677, 1000, 1000] + +f = h5py.File('verysimple.nx5', 'w') +f.attrs['default'] = 'entry' + +nxentry = f.create_group('entry') +nxentry.attrs["NX_class"] = 'NXentry' +nxentry.attrs['default'] = 'data' + +nxdata = nxentry.create_group('data') +nxdata.attrs["NX_class"] = 'NXdata' +nxdata.attrs['signal'] = 'counts' +nxdata.attrs['axes'] = 'two_theta' +nxdata.attrs['two_theta_indices'] = [0,] + +tth = nxdata.create_dataset('two_theta', data=angle) +tth.attrs['units'] = 'degrees' +tth.attrs['long_name'] = 'two_theta (degrees)' + +counts = nxdata.create_dataset('counts', data=diode) +counts.attrs['units'] = 'counts' +counts.attrs['long_name'] = 'photodiode counts' f.close() diff --git a/definitions/utils/build_preparation.py b/definitions/utils/build_preparation.py index 83108ab37..923c57b2b 100644 --- a/definitions/utils/build_preparation.py +++ b/definitions/utils/build_preparation.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -""" +''' Copy all resources for out-of-source documentation build Since we provide a build for Linux, MacOSX, and Windows, @@ -16,7 +16,7 @@ Here, we identify and copy all resources to build. The target directory is assumed to be the current directory. -""" +''' # Re-run this code to bring in any changed files (for incremental build) # Be sure to properly specify the source and target directories. @@ -25,35 +25,35 @@ from local_utilities import replicate -MTIME_TOLERANCE = 0.001 # ignore mtime differences <= 1 ms +MTIME_TOLERANCE = 0.001 # ignore mtime differences <= 1 ms ROOT_DIR_EXPECTED_RESOURCES = { - "files": """COPYING LGPL.txt Makefile NXDL_VERSION + 'files': '''COPYING LGPL.txt Makefile NXDL_VERSION nxdl.xsd nxdlTypes.xsd README.md - """.split(), - "subdirs": """applications base_classes contributed_definitions manual + '''.split(), + 'subdirs': '''applications base_classes contributed_definitions manual package utils www impatient-guide - """.split(), + '''.split(), } -REPLICATED_RESOURCES = """ +REPLICATED_RESOURCES = ''' LGPL.txt Makefile nxdl.xsd nxdlTypes.xsd NXDL_VERSION base_classes applications contributed_definitions manual utils impatient-guide -""".split() +'''.split() def mtime_size(filename): - """get the modification time and size of the given item""" + '''get the modification time and size of the given item''' file_status = os.stat(filename) return file_status.st_mtime, file_status.st_size def standardize_name(path, resource_name): - """always use the absolute path to the filesystem resource""" + '''always use the absolute path to the filesystem resource''' return os.path.abspath(os.path.join(path, resource_name)) def identical(source, target): - """compare if the resource is the same on both paths""" + '''compare if the resource is the same on both paths''' if not os.path.exists(target): return False s_mtime, s_size = mtime_size(source) @@ -62,7 +62,7 @@ def identical(source, target): def get_source_items(resources, source_path): - """walk the source_path directories accumulating files to be checked""" + '''walk the source_path directories accumulating files to be checked''' file_list = [] path_list = [] for path in sorted(resources): @@ -77,7 +77,7 @@ def get_source_items(resources, source_path): def is_definitions_directory(basedir): - """test if ``basedir`` is a NeXus definitions directory""" + '''test if ``basedir`` is a NeXus definitions directory''' # look for the expected files and subdirectories in the root directory for item_list in ROOT_DIR_EXPECTED_RESOURCES.values(): for item in item_list: @@ -87,79 +87,76 @@ def is_definitions_directory(basedir): def qualify_inputs(source_dir, target_path): - """raise error if this program cannot continue, based on the inputs""" + '''raise error if this program cannot continue, based on the inputs''' if not os.path.exists(source_dir): - raise RuntimeError("Cannot find " + source_dir) + raise RuntimeError('Cannot find ' + source_dir) if not os.path.isdir(source_dir): - raise RuntimeError("Not a directory: " + source_dir) + raise RuntimeError('Not a directory: ' + source_dir) if not is_definitions_directory(source_dir): - msg = "Not a NeXus definitions root directory " + source_dir + msg = 'Not a NeXus definitions root directory ' + source_dir raise RuntimeError(msg) - + if source_dir == target_path: - msg = "Source and target directories cannot be the same" + msg = 'Source and target directories cannot be the same' raise RuntimeError(msg) def command_args(): - """get the command-line arguments, handle syntax errors""" + '''get the command-line arguments, handle syntax errors''' import argparse - doc = __doc__.strip().splitlines()[0] parser = argparse.ArgumentParser(prog=sys.argv[0], description=doc) - parser.add_argument( - "defs_dir", action="store", help="path to NeXus definitions root directory" - ) - parser.add_argument( - "build_dir", - action="store", - default=None, - nargs="?", - help="path to target directory (default: current directory)", - ) + parser.add_argument('defs_dir', + action='store', + help="path to NeXus definitions root directory") + parser.add_argument('build_dir', + action='store', + default=None, + nargs='?', + help="path to target directory (default: current directory)") return parser.parse_args() def update(source_path, target_path): - """ + ''' duplicate directory from source_path to target_path - + :param source_path str: source directory (NeXus definitions dir) :param target_path str: target directory is specified for build product - """ + ''' # TODO: what about file items in target_path that are not in source_path? source_path = os.path.abspath(source_path) target_path = os.path.abspath(target_path) qualify_inputs(source_path, target_path) - + paths, files = get_source_items(REPLICATED_RESOURCES, source_path) - print("source has %d directories and %d files" % (len(paths), len(files))) - + print('source has %d directories and %d files' % (len(paths), len(files))) + # create all the directories / subdirectories for source in sorted(paths): - relative_name = source[len(source_path) :].lstrip(os.sep) + relative_name = source[len(source_path):].lstrip(os.sep) target = standardize_name(target_path, relative_name) if not os.path.exists(target): - print("create directory %s" % target) + print('create directory %s' % target) os.mkdir(target, os.stat(source_path).st_mode) # check if the files need to be updated for source in sorted(files): - relative_name = source[len(source_path) :].lstrip(os.sep) + relative_name = source[len(source_path):].lstrip(os.sep) target = standardize_name(target_path, relative_name) if not identical(source, target): - print("update file %s" % target) + print('update file %s' % target) replicate(source, target) def main(): - """ + ''' standard command-line processing - + source directory (NeXus definitions dir) named as command line argument target directory is specified (or defaults to present working directory) - """ + ''' cli = command_args() source_path = os.path.abspath(cli.defs_dir) target_path = cli.build_dir or os.path.abspath(os.getcwd()) @@ -167,24 +164,23 @@ def main(): def __developer_build_setup__(): - """for use with source-code debugger ONLY""" + '''for use with source-code debugger ONLY''' import shutil - # sys.argv.append('-h') - os.chdir("../") - os.chdir("build") - sys.argv.append("..") + os.chdir('../') + os.chdir('build') + sys.argv.append('..') -if __name__ == "__main__": +if __name__ == '__main__': # __developer_build_setup__() main() # NeXus - Neutron and X-ray Common Data Format -# +# # Copyright (C) 2008-2015 NeXus International Advisory Committee (NIAC) -# +# # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either diff --git a/definitions/utils/create_release_notes.py b/definitions/utils/create_release_notes.py index 342c5da37..7fe7eb253 100755 --- a/definitions/utils/create_release_notes.py +++ b/definitions/utils/create_release_notes.py @@ -4,14 +4,14 @@ Create release notes for a new release of this GitHub repository. """ -# Requires: +# Requires: # # * assumes current directory is within a repository clone # * pyGithub (conda or pip install) - https://pygithub.readthedocs.io/ # * Github personal access token (https://github.com/settings/tokens) # -# Github token access is needed or the GitHub API limit -# will likely interfere with making a complete report +# Github token access is needed or the GitHub API limit +# will likely interfere with making a complete report # of the release. import argparse @@ -22,26 +22,26 @@ logging.basicConfig(level=logging.WARNING) -logger = logging.getLogger("create_release_notes") +logger = logging.getLogger('create_release_notes') def findGitConfigFile(): """ return full path to .git/config file - + must be in current working directory or some parent directory - - This is a simplistic search that could be improved by using + + This is a simplistic search that could be improved by using an open source package. - + Needs testing for when things are wrong. """ path = os.getcwd() for i in range(99): config_file = os.path.join(path, ".git", "config") if os.path.exists(config_file): - return config_file # found it! - + return config_file # found it! + # next, look in the parent directory path = os.path.abspath(os.path.join(path, "..")) @@ -49,28 +49,26 @@ def findGitConfigFile(): logger.error(msg) raise ValueError(msg) - def parse_git_url(url): """ return (organization, repository) tuple from url line of .git/config file """ - if url.startswith("git@"): # deal with git@github.com:org/repo.git + if url.startswith("git@"): # deal with git@github.com:org/repo.git url = url.split(":")[1] org, repo = url.rstrip(".git").split("/")[-2:] return org, repo - def getRepositoryInfo(): """ return (organization, repository) tuple from .git/config file - - This is a simplistic search that could be improved by using + + This is a simplistic search that could be improved by using an open source package. - + Needs testing for when things are wrong. """ config_file = findGitConfigFile() - + with open(config_file, "r") as f: for line in f.readlines(): line = line.strip() @@ -82,11 +80,10 @@ def getRepositoryInfo(): raise ValueError(msg) return parse_git_url(url) - def get_release_info(token, base_tag_name, head_branch_name, milestone_name): """mine the Github API for information about this release""" organization_name, repository_name = getRepositoryInfo() - gh = github.Github(token) # GitHub Personal Access Token + gh = github.Github(token) # GitHub Personal Access Token user = gh.get_user(organization_name) logger.debug(f"user: {user}") @@ -95,7 +92,9 @@ def get_release_info(token, base_tag_name, head_branch_name, milestone_name): logger.debug(f"repo: {repo}") milestones = [ - m for m in repo.get_milestones(state="all") if m.title == milestone_name + m + for m in repo.get_milestones(state="all") + if m.title == milestone_name ] if len(milestones) == 0: msg = f"Could not find milestone: {milestone_name}" @@ -125,14 +124,20 @@ def get_release_info(token, base_tag_name, head_branch_name, milestone_name): logger.debug(f"# tags: {len(tags)}") pulls = { - p.number: p for p in repo.get_pulls(state="closed") if p.closed_at > earliest + p.number: p + for p in repo.get_pulls(state="closed") + if p.closed_at > earliest } logger.debug(f"# pulls: {len(pulls)}") issues = { i.number: i for i in repo.get_issues(milestone=milestone, state="closed") - if ((milestone is not None or i.closed_at > earliest) and i.number not in pulls) + if ( + (milestone is not None or i.closed_at > earliest) + and + i.number not in pulls + ) } logger.debug(f"# issues: {len(issues)}") @@ -145,36 +150,43 @@ def parse_command_line(): parser = argparse.ArgumentParser(description=doc) help_text = "name of tag to start the range" - parser.add_argument("base", action="store", help=help_text) + parser.add_argument('base', action='store', help=help_text) help_text = "name of milestone" - parser.add_argument("milestone", action="store", help=help_text) + parser.add_argument('milestone', action='store', help=help_text) parser.add_argument( - "token", - action="store", - help=("personal access token " "(see: https://github.com/settings/tokens)"), - ) + 'token', + action='store', + help=( + "personal access token " + "(see: https://github.com/settings/tokens)")) help_text = "name of tag, branch, SHA to end the range" help_text += ' (default="main")' parser.add_argument( - "--head", action="store", dest="head", nargs="?", help=help_text, default="main" - ) + "--head", + action='store', + dest='head', + nargs='?', + help = help_text, + default="main") return parser.parse_args() def str2time(time_string): """convert date/time string to datetime object - + input string example: ``Tue, 20 Dec 2016 17:35:40 GMT`` """ if time_string is None: msg = f"need valid date/time string, not: {time_string}" logger.error(msg) raise ValueError(msg) - return datetime.datetime.strptime(time_string, "%a, %d %b %Y %H:%M:%S %Z") + return datetime.datetime.strptime( + time_string, + "%a, %d %b %Y %H:%M:%S %Z") def report(title, repo, milestone, tags, pulls, issues, commits): @@ -187,7 +199,7 @@ def report(title, repo, milestone, tags, pulls, issues, commits): print(f"* **milestone**: [{milestone.title}]({milestone.url})") print("") print("section | quantity") - print("-" * 5, " | ", "-" * 5) + print("-"*5, " | ", "-"*5) print(f"[New Tags](#tags) | {len(tags)}") print(f"[Pull Requests](#pull-requests) | {len(pulls)}") print(f"[Issues](#issues) | {len(issues)}") @@ -199,7 +211,7 @@ def report(title, repo, milestone, tags, pulls, issues, commits): print("-- none --") else: print("tag | date | name") - print("-" * 5, " | ", "-" * 5, " | ", "-" * 5) + print("-"*5, " | ", "-"*5, " | ", "-"*5) for k, tag in sorted(tags.items()): commit = repo.get_commit(tag.commit.sha) when = str2time(commit.last_modified).strftime("%Y-%m-%d") @@ -211,13 +223,11 @@ def report(title, repo, milestone, tags, pulls, issues, commits): print("-- none --") else: print("pull request | date | state | title") - print("-" * 5, " | ", "-" * 5, " | ", "-" * 5, " | ", "-" * 5) + print("-"*5, " | ", "-"*5, " | ", "-"*5, " | ", "-"*5) for k, pull in sorted(pulls.items()): state = {True: "merged", False: "closed"}[pull.merged] when = str2time(pull.last_modified).strftime("%Y-%m-%d") - print( - f"[#{pull.number}]({pull.html_url}) | {when} | {state} | {pull.title}" - ) + print(f"[#{pull.number}]({pull.html_url}) | {when} | {state} | {pull.title}") print("") print("### Issues") print("") @@ -225,7 +235,7 @@ def report(title, repo, milestone, tags, pulls, issues, commits): print("-- none --") else: print("issue | date | title") - print("-" * 5, " | ", "-" * 5, " | ", "-" * 5) + print("-"*5, " | ", "-"*5, " | ", "-"*5) for k, issue in sorted(issues.items()): if k not in pulls: when = issue.closed_at.strftime("%Y-%m-%d") @@ -237,10 +247,10 @@ def report(title, repo, milestone, tags, pulls, issues, commits): print("-- none --") else: print("commit | date | message") - print("-" * 5, " | ", "-" * 5, " | ", "-" * 5) + print("-"*5, " | ", "-"*5, " | ", "-"*5) for k, commit in commits.items(): message = commit.commit.message.splitlines()[0] - when = commit.raw_data["commit"]["committer"]["date"].split("T")[0] + when = commit.raw_data['commit']['committer']['date'].split("T")[0] print(f"[{k[:7]}]({commit.html_url}) | {when} | {message}") @@ -258,19 +268,20 @@ def main(base=None, head=None, milestone=None, token=None, debug=False): token = cmd.token logger.setLevel(logging.WARNING) - info = get_release_info(token, base_tag_name, head_branch_name, milestone_name) + info = get_release_info( + token, base_tag_name, head_branch_name, milestone_name) # milestone, repo, tags, pulls, issues, commits = info report(milestone_name, *info) -if __name__ == "__main__": +if __name__ == '__main__': main() # NeXus - Neutron and X-ray Common Data Format -# +# # Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# +# # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either diff --git a/definitions/utils/dev_create_release_notes.py b/definitions/utils/dev_create_release_notes.py index 87186f2fb..613619fcd 100755 --- a/definitions/utils/dev_create_release_notes.py +++ b/definitions/utils/dev_create_release_notes.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -""" +''' Developers: use this code to develop and test create_release_notes.py -""" +''' import os CREDS_FILE = os.path.join( @@ -15,5 +15,9 @@ token = cf.read().strip() from create_release_notes import main - -main(base="v2018.5", head="main", milestone="NXDL 2020.1", token=token, debug=True) +main( + base="v2018.5", + head="main", + milestone="NXDL 2020.1", + token=token, + debug=True) diff --git a/definitions/utils/dev_nxdl2rst.py b/definitions/utils/dev_nxdl2rst.py index 498016d89..fa8a41a05 100755 --- a/definitions/utils/dev_nxdl2rst.py +++ b/definitions/utils/dev_nxdl2rst.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -""" +''' Developers: use this code to develop and test nxdl2rst.py -""" +''' # testing: # cd /tmp @@ -27,7 +27,7 @@ # nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXcrystal.nxdl.xml') # nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXentry.nxdl.xml') # nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXobject.nxdl.xml') -nxdl = os.path.join(BASEDIR, "..", "base_classes", "NXroot.nxdl.xml") +nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXroot.nxdl.xml') # nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXuser.nxdl.xml') # nxdl = os.path.join(BASEDIR, '..', 'contributed_definitions', 'NXarpes.nxdl.xml') # nxdl = os.path.join(BASEDIR, '..', 'contributed_definitions', 'NXmagnetic_kicker.nxdl.xml') diff --git a/definitions/utils/dev_units2rst.py b/definitions/utils/dev_units2rst.py index 767f2c537..601619f87 100755 --- a/definitions/utils/dev_units2rst.py +++ b/definitions/utils/dev_units2rst.py @@ -1,12 +1,12 @@ #!/usr/bin/env python -""" +''' Developers: use this code to develop and test nxdl2rst.py -""" +''' import sys from units2rst import worker sys.argv.append("../nxdlTypes.xsd") -worker("anyUnitsAttr") +worker('anyUnitsAttr') diff --git a/definitions/utils/local_utilities.py b/definitions/utils/local_utilities.py index 8d2750d92..af4979a61 100644 --- a/definitions/utils/local_utilities.py +++ b/definitions/utils/local_utilities.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -""" +''' Common code for NeXus definitions Python tools ====================== ================================== @@ -12,57 +12,58 @@ :meth:`replicate_tree` copy directory stack ====================== ================================== -""" +''' import os, sys, re import shutil + def printf(str, *args): - """formatted print without automatic newline""" - print(str % args, end="") + '''formatted print without automatic newline''' + print(str % args, end='') def mtime(file_name): - """return file modification time""" + '''return file modification time''' return os.stat(file_name)[stat.ST_MTIME] def replicate(source, target): - """ + ''' for directories or files: copy ``source`` to ``target``, replaces ``target`` - + :param str source: path to source resource :param str target: path to target location - """ + ''' if os.path.isfile(source): shutil.copy2(source, target) elif os.path.isdir(source): replicate_tree(source, target) else: - msg = "Do not know how to copy (skipped): " + source + msg = 'Do not know how to copy (skipped): ' + source raise RuntimeWarning(msg) def replicate_tree(source, target): - """ + ''' for directories: copy ``source`` to ``target``, replaces ``target`` - + :param str source: path to source resource (a directory) :param str target: path to target location (a directory) - """ + ''' if os.path.exists(source): if os.path.exists(target): shutil.rmtree(target, ignore_errors=True) shutil.copytree(source, target) else: - raise RuntimeError("Directory not found: " + source) + raise RuntimeError('Directory not found: ' + source) # NeXus - Neutron and X-ray Common Data Format -# +# # Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# +# # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either diff --git a/definitions/utils/nxdl2rst.py b/definitions/utils/nxdl2rst.py index fb308aa11..954d6f314 100755 --- a/definitions/utils/nxdl2rst.py +++ b/definitions/utils/nxdl2rst.py @@ -1,10 +1,10 @@ #!/usr/bin/env python -""" +''' Read the NeXus NXDL class specification and describe it. Write a restructured text (.rst) document for use in the NeXus manual in the NeXus NXDL Classes chapter. -""" +''' # testing: see file dev_nxdl2rst.py @@ -18,7 +18,7 @@ from local_utilities import printf, replicate -INDENTATION_UNIT = " " +INDENTATION_UNIT = ' ' listing_category = None anchor_list = [] # list of all hypertext anchors @@ -65,165 +65,156 @@ def sorter(key): print(table) -def fmtTyp(node): - typ = node.get("type", ":ref:`NX_CHAR `") # per default - if typ.startswith("NX_"): - typ = ":ref:`%s <%s>`" % (typ, typ) +def fmtTyp( node ): + typ = node.get('type', ':ref:`NX_CHAR `') # per default + if typ.startswith('NX_'): + typ = ':ref:`%s <%s>`' % (typ, typ) return typ -def fmtUnits(node): - units = node.get("units", "") +def fmtUnits( node ): + units = node.get('units', '') if not units: - return "" - if units.startswith("NX_"): - units = "\ :ref:`%s <%s>`" % (units, units) - return " {units=%s}" % units + return '' + if units.startswith('NX_'): + units = '\ :ref:`%s <%s>`' % (units, units) + return ' {units=%s}' % units -def getDocBlocks(ns, node): - docnodes = node.xpath("nx:doc", namespaces=ns) - if docnodes is None or len(docnodes) == 0: - return "" +def getDocBlocks( ns, node ): + docnodes = node.xpath('nx:doc', namespaces=ns) + if docnodes is None or len(docnodes)==0: + return '' if len(docnodes) > 1: - raise Exception( - "Too many doc elements: line %d, %s" - % (node.sourceline, os.path.split(node.base)[1]) - ) + raise Exception( 'Too many doc elements: line %d, %s' % + (node.sourceline, os.path.split(node.base)[1]) ) docnode = docnodes[0] # be sure to grab _all_ content in the documentation # it might look like XML - s = lxml.etree.tostring( - docnode, pretty_print=True, method="c14n", with_comments=False - ).decode("utf-8") - m = re.search(r"^]*>\n?(.*)\n?$", s, re.DOTALL) + s = lxml.etree.tostring(docnode, pretty_print=True, + method='c14n', with_comments=False).decode('utf-8') + m = re.search(r'^]*>\n?(.*)\n?$', s, re.DOTALL ) if not m: - raise Exception("unexpected docstring [%s] " % s) + raise Exception( 'unexpected docstring [%s] ' % s ) text = m.group(1) # substitute HTML entities in markup: "<" for "<" # thanks: http://stackoverflow.com/questions/2087370/decode-html-entities-in-python-string htmlparser = HTMLParser.HTMLParser() - try: # see #661 + try: # see #661 import html - text = html.unescape(text) except (ImportError, AttributeError): text = htmlparser.unescape(text) # Blocks are separated by whitelines - blocks = re.split("\n\s*\n", text) - if len(blocks) == 1 and len(blocks[0].splitlines()) == 1: - return [blocks[0].rstrip().lstrip()] + blocks = re.split( '\n\s*\n', text ) + if len(blocks)==1 and len(blocks[0].splitlines())==1: + return [ blocks[0].rstrip().lstrip() ] # Indentation must be given by first line - m = re.match(r"(\s*)(\S+)", blocks[0]) + m = re.match(r'(\s*)(\S+)', blocks[0]) if not m: - return [""] + return [ '' ] indent = m.group(1) # Remove common indentation as determined from first line - if indent == "": - raise Exception( - "Missing initial indentation in of %s [%s]" - % (node.get("name"), blocks[0]) - ) + if indent=="": + raise Exception( 'Missing initial indentation in of %s [%s]' % + ( node.get('name'), blocks[0] ) ) out_blocks = [] for block in blocks: lines = block.rstrip().splitlines() out_lines = [] for line in lines: - if line[: len(indent)] != indent: - raise Exception( - 'Bad indentation in of %s [%s]: expected "%s" found "%s".' - % ( - node.get("name"), - block, - re.sub(r"\t", "\\\\t", indent), - re.sub(r"\t", "\\\\t", line), - ) - ) - out_lines.append(line[len(indent) :]) - out_blocks.append("\n".join(out_lines)) + if line[:len(indent)]!=indent: + raise Exception( 'Bad indentation in of %s [%s]: expected "%s" found "%s".' % + ( node.get('name'), block, + re.sub(r'\t',"\\\\t", indent ), + re.sub(r'\t',"\\\\t", line ), + ) ) + out_lines.append( line[len(indent):] ) + out_blocks.append( "\n".join(out_lines) ) return out_blocks -def getDocLine(ns, node): - blocks = getDocBlocks(ns, node) - if len(blocks) == 0: - return "" - if len(blocks) > 1: - raise Exception("Unexpected multi-paragraph doc [%s]" % "|".join(blocks)) - return re.sub(r"\n", " ", blocks[0]) +def getDocLine( ns, node ): + blocks = getDocBlocks( ns, node ) + if len(blocks)==0: + return '' + if len(blocks)>1: + raise Exception( 'Unexpected multi-paragraph doc [%s]' % + '|'.join(blocks) ) + return re.sub(r'\n', " ", blocks[0]) def get_minOccurs(node, use_application_defaults): - """ + ''' get the value for the ``minOccurs`` attribute :param obj node: instance of lxml.etree._Element :param bool use_application_defaults: use special case value :returns str: value of the attribute (or its default) - """ + ''' # TODO: can we improve on the default by exmaining nxdl.xsd? - minOccurs_default = {True: "1", False: "0"}[use_application_defaults] - minOccurs = node.get("minOccurs", minOccurs_default) + minOccurs_default = {True: '1', False: '0'}[use_application_defaults] + minOccurs = node.get('minOccurs', minOccurs_default) return minOccurs def get_required_or_optional_text(node, use_application_defaults): - """ + ''' make clear if a reported item is required or optional :param obj node: instance of lxml.etree._Element :param bool use_application_defaults: use special case value :returns: formatted text - """ - tag = node.tag.split("}")[-1] - nm = node.get("name") - if tag in ("field", "group"): + ''' + tag = node.tag.split('}')[-1] + nm = node.get('name') + if tag in ('field', 'group'): optional_default = not use_application_defaults - optional = node.get("optional", optional_default) in (True, "true", "1", 1) - recommended = node.get("recommended", None) in (True, "true", "1", 1) + optional = node.get('optional', optional_default) in (True, 'true', '1', 1) + recommended = node.get('recommended', None) in (True, 'true', '1', 1) minOccurs = get_minOccurs(node, use_application_defaults) - if minOccurs in ("0", 0) or optional: - optional_text = "(optional) " + if minOccurs in ('0', 0) or optional: + optional_text = '(optional) ' elif recommended: - optional_text = "(recommended) " - elif minOccurs in ("1", 1): - optional_text = "(required) " + optional_text = '(recommended) ' + elif minOccurs in ('1', 1): + optional_text = '(required) ' else: # this is unexpected and remarkable # TODO: add a remark to the log - optional_text = "(``minOccurs=%s``) " % str(minOccurs) - elif tag in ("attribute",): + optional_text = '(``minOccurs=%s``) ' % str(minOccurs) + elif tag in ('attribute',): optional_default = not use_application_defaults - optional = node.get("optional", optional_default) in (True, "true", "1", 1) - recommended = node.get("recommended", None) in (True, "true", "1", 1) - optional_text = {True: "(optional) ", False: "(required) "}[optional] + optional = node.get('optional', optional_default) in (True, 'true', '1', 1) + recommended = node.get('recommended', None) in (True, 'true', '1', 1) + optional_text = {True: '(optional) ', False: '(required) '}[optional] if recommended: - optional_text = "(recommended) " + optional_text = '(recommended) ' else: - optional_text = "(unknown tag: " + str(tag) + ") " + optional_text = '(unknown tag: ' + str(tag) + ') ' return optional_text -def analyzeDimensions(ns, parent): - node_list = parent.xpath("nx:dimensions", namespaces=ns) +def analyzeDimensions( ns, parent ): + node_list = parent.xpath('nx:dimensions', namespaces=ns) if len(node_list) != 1: - return "" + return '' node = node_list[0] # rank = node.get('rank') # ignore this - node_list = node.xpath("nx:dim", namespaces=ns) + node_list = node.xpath('nx:dim', namespaces=ns) dims = [] for subnode in node_list: - value = subnode.get("value") + value = subnode.get('value') if not value: - value = "ref(%s)" % subnode.get("ref") - dims.append(value) - return "[%s]" % (", ".join(dims)) + value = 'ref(%s)' % subnode.get('ref') + dims.append( value ) + return '[%s]' % ( ', '.join(dims) ) def hyperlinkTarget(parent_path, name, nxtype): @@ -232,83 +223,85 @@ def hyperlinkTarget(parent_path, name, nxtype): sep = "@" else: sep = "/" - target = "%s%s%s-%s" % (parent_path, sep, name, nxtype) + target = "%s%s%s-%s" % ( + parent_path, sep, name, nxtype + ) addAnchor(target) return ".. _%s:\n" % target -def printEnumeration(indent, ns, parent): - node_list = parent.xpath("nx:item", namespaces=ns) +def printEnumeration( indent, ns, parent ): + node_list = parent.xpath('nx:item', namespaces=ns) if len(node_list) == 0: - return "" + return '' if len(node_list) == 1: - printf("%sObligatory value:", indent) + printf('%sObligatory value:', indent) else: - printf("%sAny of these values:", indent) + printf('%sAny of these values:', indent) docs = OrderedDict() for item in node_list: - name = item.get("value") + name = item.get('value') docs[name] = getDocLine(ns, item) ENUMERATION_INLINE_LENGTH = 60 - def show_as_typed_text(msg): - return "``%s``" % msg - - oneliner = " | ".join(map(show_as_typed_text, docs.keys())) - if any(doc for doc in docs.values()) or len(oneliner) > ENUMERATION_INLINE_LENGTH: + return '``%s``' % msg + oneliner = ' | '.join( map(show_as_typed_text, docs.keys()) ) + if ( any( doc for doc in docs.values() ) or + len( oneliner ) > ENUMERATION_INLINE_LENGTH ): # print one item per line - print("\n") + print('\n') for name, doc in docs.items(): - printf("%s * %s", indent, show_as_typed_text(name)) + printf('%s * %s', indent, show_as_typed_text(name)) if doc: - printf(": %s", doc) - print("\n") + printf(': %s', doc) + print('\n') else: # print all items in one line - print(" %s" % (oneliner)) - print("") + print(' %s' % ( oneliner ) ) + print('') -def printDoc(indent, ns, node, required=False): +def printDoc( indent, ns, node, required=False): blocks = getDocBlocks(ns, node) - if len(blocks) == 0: + if len(blocks)==0: if required: - raise Exception("No documentation for: " + node.get("name")) - print("") + raise Exception( 'No documentation for: ' + node.get('name') ) + print('') else: for block in blocks: for line in block.splitlines(): - print("%s%s" % (indent, line)) + print( '%s%s' % ( indent, line ) ) print() -def printAttribute(ns, kind, node, optional, indent, parent_path): - name = node.get("name") +def printAttribute( ns, kind, node, optional, indent, parent_path ): + name = node.get('name') index_name = name - print("%s%s" % (indent, hyperlinkTarget(parent_path, name, "attribute"))) - print("%s.. index:: %s (%s attribute)\n" % (indent, index_name, kind)) - print( - "%s**@%s**: %s%s%s\n" % (indent, name, optional, fmtTyp(node), fmtUnits(node)) + print("%s%s" % (indent, hyperlinkTarget(parent_path, name, 'attribute')) ) - printDoc(indent + INDENTATION_UNIT, ns, node) - node_list = node.xpath("nx:enumeration", namespaces=ns) + print( '%s.. index:: %s (%s attribute)\n' % + ( indent, index_name, kind ) ) + print( '%s**@%s**: %s%s%s\n' % ( + indent, name, optional, fmtTyp(node), fmtUnits(node) ) ) + printDoc(indent+INDENTATION_UNIT, ns, node) + node_list = node.xpath('nx:enumeration', namespaces=ns) if len(node_list) == 1: - printEnumeration(indent + INDENTATION_UNIT, ns, node_list[0]) + printEnumeration( indent+INDENTATION_UNIT, ns, node_list[0] ) -def printIfDeprecated(ns, node, indent): - deprecated = node.get("deprecated", None) +def printIfDeprecated( ns, node, indent ): + deprecated = node.get('deprecated', None) if deprecated is not None: - print("\n%s.. index:: deprecated\n" % indent) - fmt = "\n%s**DEPRECATED**: %s\n" - print(fmt % (indent, deprecated)) + print( '\n%s.. index:: deprecated\n' % indent) + fmt = '\n%s**DEPRECATED**: %s\n' + print( fmt % (indent, deprecated ) ) def printFullTree(ns, parent, name, indent, parent_path): - """ + ''' recursively print the full tree structure :param dict ns: dictionary of namespaces for use in XPath expressions @@ -316,90 +309,72 @@ def printFullTree(ns, parent, name, indent, parent_path): :param str name: name of elements, such as NXentry/NXuser :param indent: to keep track of indentation level :param parent_path: NX class path of parent nodes - """ + ''' global listing_category use_application_defaults = listing_category in ( - "application definition", - "contributed definition", - ) + 'application definition', + 'contributed definition') - for node in parent.xpath("nx:field", namespaces=ns): - name = node.get("name") + for node in parent.xpath('nx:field', namespaces=ns): + name = node.get('name') index_name = name dims = analyzeDimensions(ns, node) optional_text = get_required_or_optional_text(node, use_application_defaults) - print("%s%s" % (indent, hyperlinkTarget(parent_path, name, "field"))) - print("%s.. index:: %s (field)\n" % (indent, index_name)) + print("%s%s" % (indent, hyperlinkTarget(parent_path, name, 'field'))) + print( '%s.. index:: %s (field)\n' % + ( indent, index_name ) ) print( - "%s**%s%s**: %s%s%s\n" - % (indent, name, dims, optional_text, fmtTyp(node), fmtUnits(node)) - ) + '%s**%s%s**: %s%s%s\n' % ( + indent, name, dims, optional_text, fmtTyp(node), fmtUnits(node) + )) - printIfDeprecated(ns, node, indent + INDENTATION_UNIT) - printDoc(indent + INDENTATION_UNIT, ns, node) + printIfDeprecated( ns, node, indent+INDENTATION_UNIT ) + printDoc(indent+INDENTATION_UNIT, ns, node) - node_list = node.xpath("nx:enumeration", namespaces=ns) + node_list = node.xpath('nx:enumeration', namespaces=ns) if len(node_list) == 1: - printEnumeration(indent + INDENTATION_UNIT, ns, node_list[0]) + printEnumeration( indent+INDENTATION_UNIT, ns, node_list[0] ) - for subnode in node.xpath("nx:attribute", namespaces=ns): + for subnode in node.xpath('nx:attribute', namespaces=ns): optional = get_required_or_optional_text(subnode, use_application_defaults) - printAttribute( - ns, - "field", - subnode, - optional, - indent + INDENTATION_UNIT, - parent_path + "/" + name, - ) + printAttribute( ns, 'field', subnode, optional, indent+INDENTATION_UNIT, parent_path+"/"+name ) - for node in parent.xpath("nx:group", namespaces=ns): - name = node.get("name", "") - typ = node.get("type", "untyped (this is an error; please report)") + for node in parent.xpath('nx:group', namespaces=ns): + name = node.get('name', '') + typ = node.get('type', 'untyped (this is an error; please report)') optional_text = get_required_or_optional_text(node, use_application_defaults) - if typ.startswith("NX"): - if name == "": - name = typ.lstrip("NX").upper() - typ = ":ref:`%s`" % typ - print("%s%s" % (indent, hyperlinkTarget(parent_path, name, "group"))) - print("%s**%s**: %s%s\n" % (indent, name, optional_text, typ)) + if typ.startswith('NX'): + if name == '': + name = typ.lstrip('NX').upper() + typ = ':ref:`%s`' % typ + print("%s%s" % (indent, hyperlinkTarget(parent_path, name, 'group'))) + print( '%s**%s**: %s%s\n' % (indent, name, optional_text, typ ) ) - printIfDeprecated(ns, node, indent + INDENTATION_UNIT) - printDoc(indent + INDENTATION_UNIT, ns, node) + printIfDeprecated(ns, node, indent+INDENTATION_UNIT) + printDoc(indent+INDENTATION_UNIT, ns, node) - for subnode in node.xpath("nx:attribute", namespaces=ns): + for subnode in node.xpath('nx:attribute', namespaces=ns): optional = get_required_or_optional_text(subnode, use_application_defaults) - printAttribute( - ns, - "group", - subnode, - optional, - indent + INDENTATION_UNIT, - parent_path + "/" + name, - ) + printAttribute( ns, 'group', subnode, optional, indent+INDENTATION_UNIT, parent_path+"/"+name ) - nodename = "%s/%s" % (name, node.get("type")) - printFullTree( - ns, node, nodename, indent + INDENTATION_UNIT, parent_path + "/" + name - ) + nodename = '%s/%s' % (name, node.get('type')) + printFullTree(ns, node, nodename, indent+INDENTATION_UNIT, parent_path+"/"+name) - for node in parent.xpath("nx:link", namespaces=ns): - name = node.get("name") - print("%s%s" % (indent, hyperlinkTarget(parent_path, name, "link"))) - print( - "%s**%s**: :ref:`link` (suggested target: ``%s``)\n" - % (indent, name, node.get("target")) - ) - printDoc(indent + INDENTATION_UNIT, ns, node) + for node in parent.xpath('nx:link', namespaces=ns): + name = node.get('name') + print("%s%s" % (indent, hyperlinkTarget(parent_path, name, 'link'))) + print( '%s**%s**: :ref:`link` (suggested target: ``%s``)\n' % ( + indent, name, node.get('target') ) ) + printDoc(indent+INDENTATION_UNIT, ns, node) def print_rst_from_nxdl(nxdl_file): - """ + ''' print restructured text from the named .nxdl.xml file - """ + ''' global listing_category # parse input file into tree tree = lxml.etree.parse(nxdl_file) @@ -407,138 +382,138 @@ def print_rst_from_nxdl(nxdl_file): # The following URL is outdated, but that doesn't matter; # it won't be accessed; it's just an arbitrary namespace name. # It only needs to match the xmlns attribute in the NXDL files. - NAMESPACE = "http://definition.nexusformat.org/nxdl/3.1" - ns = {"nx": NAMESPACE} + NAMESPACE = 'http://definition.nexusformat.org/nxdl/3.1' + ns = {'nx': NAMESPACE} root = tree.getroot() - name = root.get("name") + name = root.get('name') title = name - parent_path = "/" + name # absolute path of parent nodes, no trailing / - if len(name) < 2 or name[0:2] != "NX": - raise Exception('Unexpected class name "%s"; does not start with NX' % (name)) - lexical_name = name[2:] # without padding 'NX', for indexing + parent_path = "/"+name # absolute path of parent nodes, no trailing / + if len(name)<2 or name[0:2]!='NX': + raise Exception( 'Unexpected class name "%s"; does not start with NX' % + ( name ) ) + lexical_name = name[2:] # without padding 'NX', for indexing # retrieve category from directory - # subdir = os.path.split(os.path.split(tree.docinfo.URL)[0])[1] + #subdir = os.path.split(os.path.split(tree.docinfo.URL)[0])[1] subdir = root.attrib["category"] # TODO: check for consistency with root.get('category') listing_category = { - "base": "base class", - "application": "application definition", - "contributed": "contributed definition", - }[subdir] + 'base': 'base class', + 'application': 'application definition', + 'contributed': 'contributed definition', + }[subdir] use_application_defaults = listing_category in ( - "application definition", - "contributed definition", - ) + 'application definition', + 'contributed definition') # print ReST comments and section header - print( - ".. auto-generated by script %s from the NXDL source %s" - % (sys.argv[0], sys.argv[1]) - ) - print("") - print(".. index::") - print(" ! %s (%s)" % (name, listing_category)) - print(" ! %s (%s)" % (lexical_name, listing_category)) - print(" see: %s (%s); %s" % (lexical_name, listing_category, name)) - print("") - print(".. _%s:\n" % name) - print("=" * len(title)) - print(title) - print("=" * len(title)) + print( '.. auto-generated by script %s from the NXDL source %s' % + (sys.argv[0], sys.argv[1]) ) + print('') + print( '.. index::' ) + print( ' ! %s (%s)' % (name,listing_category) ) + print( ' ! %s (%s)' % (lexical_name,listing_category) ) + print( ' see: %s (%s); %s' % + (lexical_name,listing_category, name) ) + print('') + print( '.. _%s:\n' % name ) + print( '='*len(title) ) + print( title ) + print( '='*len(title) ) # print category & parent class - extends = root.get("extends") + extends = root.get('extends') if extends is None: - extends = "none" + extends = 'none' else: - extends = ":ref:`%s`" % extends + extends = ':ref:`%s`' % extends - print("") - print("**Status**:\n") - print(" %s, extends %s" % (listing_category.strip(), extends)) + print('') + print( '**Status**:\n' ) + print( ' %s, extends %s' % + ( listing_category.strip(), + extends ) ) - printIfDeprecated(ns, root, "") + printIfDeprecated(ns, root, '') # print official description of this class - print("") - print("**Description**:\n") + print('') + print( '**Description**:\n' ) printDoc(INDENTATION_UNIT, ns, root, required=True) + # print symbol list - node_list = root.xpath("nx:symbols", namespaces=ns) - print("**Symbols**:\n") + node_list = root.xpath('nx:symbols', namespaces=ns) + print( '**Symbols**:\n' ) if len(node_list) == 0: - print(" No symbol table\n") + print( ' No symbol table\n' ) elif len(node_list) > 1: - raise Exception("Invalid symbol table in " % root.get("name")) + raise Exception( 'Invalid symbol table in ' % root.get('name') ) else: - printDoc(INDENTATION_UNIT, ns, node_list[0]) - for node in node_list[0].xpath("nx:symbol", namespaces=ns): + printDoc( INDENTATION_UNIT, ns, node_list[0] ) + for node in node_list[0].xpath('nx:symbol', namespaces=ns): doc = getDocLine(ns, node) - printf(" **%s**", node.get("name")) + printf(' **%s**', node.get('name')) if doc: - printf(": %s", doc) - print("\n") + printf(': %s', doc) + print('\n') # print group references - print("**Groups cited**:") - node_list = root.xpath("//nx:group", namespaces=ns) + print( '**Groups cited**:' ) + node_list = root.xpath('//nx:group', namespaces=ns) groups = [] for node in node_list: - g = node.get("type") - if g.startswith("NX") and g not in groups: + g = node.get('type') + if g.startswith('NX') and g not in groups: groups.append(g) if len(groups) == 0: - print(" none\n") + print( ' none\n' ) else: - out = [(":ref:`%s`" % g) for g in groups] - txt = ", ".join(sorted(out)) - print(" %s\n" % (txt)) - out = [("%s (base class); used in %s" % (g, listing_category)) for g in groups] - txt = ", ".join(out) - print(".. index:: %s\n" % (txt)) + out = [ (':ref:`%s`' % g) for g in groups ] + txt = ', '.join(sorted(out)) + print( ' %s\n' % ( txt ) ) + out = [ ('%s (base class); used in %s' % (g, listing_category)) for g in groups ] + txt = ', '.join(out) + print( '.. index:: %s\n' % ( txt ) ) # TODO: change instances of \t to proper indentation - html_root = "https://github.com/nexusformat/definitions/blob/main" + html_root = 'https://github.com/nexusformat/definitions/blob/main' # print full tree - print("**Structure**:\n") - for subnode in root.xpath("nx:attribute", namespaces=ns): + print( '**Structure**:\n' ) + for subnode in root.xpath('nx:attribute', namespaces=ns): optional = get_required_or_optional_text(subnode, use_application_defaults) - printAttribute( - ns, "file", subnode, optional, INDENTATION_UNIT, parent_path - ) # FIXME: +"/"+name ) + printAttribute( ns, 'file', subnode, optional, INDENTATION_UNIT, parent_path) # FIXME: +"/"+name ) printFullTree(ns, root, name, INDENTATION_UNIT, parent_path) printAnchorList() # print NXDL source location subdir_map = { - "base": "base_classes", - "application": "applications", - "contributed": "contributed_definitions", - } + 'base': 'base_classes', + 'application': 'applications', + 'contributed': 'contributed_definitions', + } print("") - print("**NXDL Source**:") - print(" %s/%s/%s.nxdl.xml" % (html_root, subdir_map[subdir], name)) + print( '**NXDL Source**:' ) + print( ' %s/%s/%s.nxdl.xml' % ( + html_root, subdir_map[subdir], name) ) def main(): - """ + ''' standard command-line processing - """ + ''' import argparse - - parser = argparse.ArgumentParser(description="test nxdl2rst code") - parser.add_argument("nxdl_file", help="name of NXDL file") + parser = argparse.ArgumentParser(description='test nxdl2rst code') + parser.add_argument('nxdl_file', help='name of NXDL file') results = parser.parse_args() nxdl_file = results.nxdl_file if not os.path.exists(nxdl_file): - print("Cannot find %s" % nxdl_file) + print( 'Cannot find %s' % nxdl_file ) exit() print_rst_from_nxdl(nxdl_file) @@ -547,16 +522,16 @@ def main(): # copy that subdirectory (quietly) to the pwd, such as: # contributed/NXcanSAS.nxdl.xml: cp -a contributed/canSAS ./ category = os.path.basename(os.getcwd()) - path = os.path.join("../../../../", category) + path = os.path.join('../../../../', category) basename = os.path.basename(nxdl_file) - corename = basename[2:].split(".")[0] + corename = basename[2:].split('.')[0] source = os.path.join(path, corename) if os.path.exists(source): - target = os.path.join(".", corename) + target = os.path.join('.', corename) replicate(source, target) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/definitions/utils/nxdl_desc2rst.py b/definitions/utils/nxdl_desc2rst.py index a06ee5f21..8211494bf 100755 --- a/definitions/utils/nxdl_desc2rst.py +++ b/definitions/utils/nxdl_desc2rst.py @@ -1,11 +1,11 @@ #!/usr/bin/env python -""" +''' Read the NXDL field types specification and find all the valid data types. Write a restructured text (.rst) document for use in the NeXus manual in the NXDL chapter. -""" +''' import os, sys @@ -13,12 +13,12 @@ import textwrap -TITLE_MARKERS = "- + ~ ^ * @".split() # used for underscoring section titles -INDENTATION = " " * 4 +TITLE_MARKERS = '- + ~ ^ * @'.split() # used for underscoring section titles +INDENTATION = ' '*4 ELEMENT_DICT = { - "attribute": """ + 'attribute': ''' An ``attribute`` element can *only* be a child of a ``field`` or ``group`` element. It is used to define *attribute* elements to be used and their data types @@ -26,8 +26,9 @@ For more details, see: :ref:`NXDL.data.type.attributeType` - """, - "definition": """ + ''', + + 'definition': ''' A ``definition`` element can *only* be used at the root level of an NXDL specification. Note: Due to the large number of attributes of the ``definition`` element, @@ -37,15 +38,17 @@ :ref:`NXDL.data.type.definition`, :ref:`NXDL.data.type.definitionType`, and :ref:`NXDL.data.type.definitionTypeAttr` - """, - "dimensions": """ + ''', + + 'dimensions': ''' The ``dimensions`` element describes the *shape* of an array. It is used *only* as a child of a ``field`` element. For more details, see: :ref:`NXDL.data.type.dimensionsType` - """, - "doc": """ + ''', + + 'doc': ''' A ``doc`` element can be a child of most NXDL elements. In most cases, the content of the ``doc`` element will also become part of the NeXus manual. @@ -58,8 +61,9 @@ For more details, see: :ref:`NXDL.data.type.docType` - """, - "enumeration": """ + ''', + + 'enumeration': ''' An ``enumeration`` element can *only* be a child of a ``field`` or ``attribute`` element. It is used to restrict the available choices to a predefined list, @@ -68,8 +72,9 @@ For more details, see: :ref:`NXDL.data.type.enumerationType` - """, - "field": """ + ''', + + 'field': ''' The ``field`` element provides the value of a named item. Many different attributes are available to further define the ``field``. Some of the attributes are not allowed to be used together (such as ``axes`` and ``axis``); see the documentation @@ -78,16 +83,18 @@ For more details, see: :ref:`NXDL.data.type.fieldType` - """, - "choice": """ + ''', + + 'choice': ''' A ``choice`` element is used when a named group might take one of several possible NeXus base classes. Logically, it must have at least two group children. For more details, see: :ref:`NXDL.data.type.choiceType` - """, - "group": """ + ''', + + 'group': ''' A ``group`` element can *only* be a child of a ``definition`` or ``group`` element. It describes a common level of organization in a NeXus data file, similar @@ -95,8 +102,9 @@ For more details, see: :ref:`NXDL.data.type.groupType` - """, - "link": """ + ''', + + 'link': ''' .. index:: single: link target @@ -109,26 +117,27 @@ For more details, see: :ref:`NXDL.data.type.linkType` - """, - "symbols": """ + ''', + + 'symbols': ''' A ``symbols`` element can *only* be a child of a ``definition`` element. It defines the array index symbols to be used when defining arrays as ``field`` elements with common dimensions and lengths. For more details, see: :ref:`NXDL.data.type.symbolsType` - """, -} + ''', + } DATATYPE_DICT = { - "basicComponent": """/xs:schema//xs:complexType[@name='basicComponent']""", - "validItemName": """/xs:schema//xs:simpleType[@name='validItemName']""", - "validNXClassName": """/xs:schema//xs:simpleType[@name='validNXClassName']""", - "validTargetName": """/xs:schema//xs:simpleType[@name='validTargetName']""", - "nonNegativeUnbounded": """/xs:schema//xs:simpleType[@name='nonNegativeUnbounded']""", -} - -ELEMENT_PREAMBLE = """ + 'basicComponent': '''/xs:schema//xs:complexType[@name='basicComponent']''', + 'validItemName': '''/xs:schema//xs:simpleType[@name='validItemName']''', + 'validNXClassName': '''/xs:schema//xs:simpleType[@name='validNXClassName']''', + 'validTargetName': '''/xs:schema//xs:simpleType[@name='validTargetName']''', + 'nonNegativeUnbounded': '''/xs:schema//xs:simpleType[@name='nonNegativeUnbounded']''', + } + +ELEMENT_PREAMBLE = ''' ============================= NXDL Elements and Field Types ============================= @@ -162,9 +171,9 @@ NXDL Elements ============= - """ + ''' -DATATYPE_PREAMBLE = """ +DATATYPE_PREAMBLE = ''' .. _NXDL.data.types.internal: @@ -177,9 +186,9 @@ or to simplify a complicated entry. While the data types are not intended for use in NXDL specifications, they define structures that may be used in NXDL specifications. -""" +''' -DATATYPE_POSTAMBLE = """ +DATATYPE_POSTAMBLE = ''' **The** ``xs:string`` **data type** The ``xs:string`` data type can contain characters, line feeds, carriage returns, and tab characters. @@ -195,202 +204,178 @@ leading and trailing spaces, and multiple spaces. See https://www.w3schools.com/xml/schema_dtypes_string.asp for more details. -""" +''' def _tagMatch(ns, parent, match_list): - """match this tag to a list""" + '''match this tag to a list''' if parent is None: raise ValueError("Must supply a valid parent node") parent_tag = parent.tag tag_found = False for item in match_list: # this routine only handles certain XML Schema components - tag_found = parent_tag == "{%s}%s" % (ns["xs"], item) + tag_found = parent_tag == '{%s}%s' % (ns['xs'], item) if tag_found: break return tag_found def _indent(indentLevel): - return INDENTATION * indentLevel + return INDENTATION*indentLevel def printTitle(title, indentLevel): print(title) - print(TITLE_MARKERS[indentLevel] * len(title) + "\n") + print(TITLE_MARKERS[indentLevel]*len(title) + '\n') def generalHandler(ns, parent=None, indentLevel=0): - """Handle XML nodes like the former XSLT template""" + '''Handle XML nodes like the former XSLT template''' # ignore things we don't know how to handle - known_tags = ("complexType", "simpleType", "group", "element", "attribute") + known_tags = ('complexType', 'simpleType', 'group', 'element', 'attribute') if not _tagMatch(ns, parent, known_tags): return - - parent_name = parent.get("name") + + parent_name = parent.get('name') if parent_name is None: return - - simple_tag = parent.tag[ - parent.tag.find("}") + 1 : - ] # cut off the namespace identifier - + + simple_tag = parent.tag[parent.tag.find('}')+1:] # cut off the namespace identifier + # ... name = parent_name # + ' data type' - if simple_tag == "attribute": - name = "@" + name - - if indentLevel == 0 and not simple_tag in ("attribute"): - print(".. index:: ! %s (NXDL data type)\n" % name) - print("\n.. _%s:\n" % ("NXDL.data.type." + name)) + if simple_tag == 'attribute': + name = '@' + name + + if indentLevel == 0 and not simple_tag in ('attribute'): + print('.. index:: ! %s (NXDL data type)\n' % name) + print('\n.. _%s:\n' % ('NXDL.data.type.'+name)) printTitle(name, indentLevel) - + printDocs(ns, parent, indentLevel) + + if len(parent.xpath('xs:attribute', namespaces=ns)) > 0: + printTitle("Attributes of "+name, indentLevel+1) + applyTemplates(ns, parent, 'xs:attribute', indentLevel+1) - if len(parent.xpath("xs:attribute", namespaces=ns)) > 0: - printTitle("Attributes of " + name, indentLevel + 1) - applyTemplates(ns, parent, "xs:attribute", indentLevel + 1) - - node_list = parent.xpath("xs:restriction", namespaces=ns) + node_list = parent.xpath('xs:restriction', namespaces=ns) if len(node_list) > 0: - # printTitle("Restrictions of "+name, indentLevel+1) - restrictionHandler(ns, node_list[0], indentLevel + 1) - node_list = parent.xpath( - "xs:simpleType/xs:restriction/xs:enumeration", namespaces=ns - ) + #printTitle("Restrictions of "+name, indentLevel+1) + restrictionHandler(ns, node_list[0], indentLevel+1) + node_list = parent.xpath('xs:simpleType/xs:restriction/xs:enumeration', namespaces=ns) if len(node_list) > 0: - # printTitle("Enumerations of "+name, indentLevel+1) - applyTemplates( - ns, - parent, - "xs:simpleType/xs:restriction", - indentLevel + 1, - handler=restrictionHandler, - ) - - if len(parent.xpath("xs:sequence/xs:element", namespaces=ns)) > 0: - printTitle("Elements of " + name, indentLevel + 1) - applyTemplates(ns, parent, "xs:sequence/xs:element", indentLevel + 1) - - node_list = parent.xpath("xs:sequence/xs:group", namespaces=ns) +# printTitle("Enumerations of "+name, indentLevel+1) + applyTemplates(ns, parent, 'xs:simpleType/xs:restriction', + indentLevel+1, handler=restrictionHandler) + + if len(parent.xpath('xs:sequence/xs:element', namespaces=ns)) > 0: + printTitle("Elements of "+name, indentLevel+1) + applyTemplates(ns, parent, 'xs:sequence/xs:element', indentLevel+1) + + node_list = parent.xpath('xs:sequence/xs:group', namespaces=ns) if len(node_list) > 0: - printTitle("Groups under " + name, indentLevel + 1) - printDocs(ns, node_list[0], indentLevel + 1) - - applyTemplates(ns, parent, "xs:simpleType", indentLevel + 1) - applyTemplates(ns, parent, "xs:complexType", indentLevel + 1) - applyTemplates(ns, parent, "xs:complexType/xs:attribute", indentLevel + 1) - applyTemplates( - ns, parent, "xs:complexContent/xs:extension/xs:attribute", indentLevel + 1 - ) - applyTemplates( - ns, parent, "xs:complexType/xs:sequence/xs:attribute", indentLevel + 1 - ) - applyTemplates(ns, parent, "xs:complexType/xs:sequence/xs:element", indentLevel + 1) - applyTemplates( - ns, - parent, - "xs:complexContent/xs:extension/xs:sequence/xs:element", - indentLevel + 1, - ) + printTitle("Groups under "+name, indentLevel+1) + printDocs(ns, node_list[0], indentLevel+1) + + applyTemplates(ns, parent, 'xs:simpleType', indentLevel+1) + applyTemplates(ns, parent, 'xs:complexType', indentLevel+1) + applyTemplates(ns, parent, 'xs:complexType/xs:attribute', indentLevel+1) + applyTemplates(ns, parent, 'xs:complexContent/xs:extension/xs:attribute', indentLevel+1) + applyTemplates(ns, parent, 'xs:complexType/xs:sequence/xs:attribute', indentLevel+1) + applyTemplates(ns, parent, 'xs:complexType/xs:sequence/xs:element', indentLevel+1) + applyTemplates(ns, parent, 'xs:complexContent/xs:extension/xs:sequence/xs:element', indentLevel+1) def restrictionHandler(ns, parent=None, indentLevel=0): - """Handle XSD restriction nodes like the former XSLT template""" - if not _tagMatch(ns, parent, ("restriction",)): + '''Handle XSD restriction nodes like the former XSLT template''' + if not _tagMatch(ns, parent, ('restriction',)): return printDocs(ns, parent, indentLevel) - print("\n") - print(_indent(indentLevel) + "The value may be any") - base = parent.get("base") - pattern_nodes = parent.xpath("xs:pattern", namespaces=ns) - enumeration_nodes = parent.xpath("xs:enumeration", namespaces=ns) + print('\n') + print(_indent(indentLevel) + 'The value may be any') + base = parent.get('base') + pattern_nodes = parent.xpath('xs:pattern', namespaces=ns) + enumeration_nodes = parent.xpath('xs:enumeration', namespaces=ns) if len(pattern_nodes) > 0: - print( - _indent(indentLevel) - + "``%s``" % base - + " that *also* matches the regular expression::\n" - ) - print(_indent(indentLevel) + " " * 4 + pattern_nodes[0].get("value")) + print(_indent(indentLevel) + '``%s``' % base + ' that *also* matches the regular expression::\n') + print(_indent(indentLevel) + ' '*4 + pattern_nodes[0].get('value')) elif len(pattern_nodes) > 0: # how will this be reached? Perhaps a deprecated procedure - print(_indent(indentLevel) + "``%s``" % base + " from this list:") + print(_indent(indentLevel) + '``%s``' % base + ' from this list:') for node in enumeration_nodes: enumerationHandler(ns, node, indentLevel) printDocs(ns, node, indentLevel) print(_indent(indentLevel)) elif len(enumeration_nodes) > 0: - print(_indent(indentLevel) + "one from this list only:\n") + print(_indent(indentLevel) + 'one from this list only:\n') for node in enumeration_nodes: enumerationHandler(ns, node, indentLevel) printDocs(ns, parent, indentLevel) print(_indent(indentLevel)) else: - print("@" + base) - print("\n") + print('@' + base) + print('\n') def enumerationHandler(ns, parent=None, indentLevel=0): - """Handle XSD enumeration nodes like the former XSLT template""" - if not _tagMatch(ns, parent, ["enumeration"]): + '''Handle XSD enumeration nodes like the former XSLT template''' + if not _tagMatch(ns, parent, ['enumeration']): return - print(_indent(indentLevel) + "* ``%s``" % parent.get("value")) + print(_indent(indentLevel) + '* ``%s``' % parent.get('value')) printDocs(ns, parent, indentLevel) def applyTemplates(ns, parent, path, indentLevel, handler=generalHandler): - """iterate the nodes found on the supplied XPath expression""" + '''iterate the nodes found on the supplied XPath expression''' db = {} for node in parent.xpath(path, namespaces=ns): - name = node.get("name") or node.get("ref") or node.get("value") + name = node.get('name') or node.get('ref') or node.get('value') if name is not None: - if name in ("nx:groupGroup",): - print(">" * 45, name) + if name in ('nx:groupGroup',): + print(">"*45, name) if name in db: raise KeyError("Duplicate name found: " + name) db[name] = node for name in sorted(db): node = db[name] handler(ns, node, indentLevel) - # printDocs(ns, node, indentLevel) + #printDocs(ns, node, indentLevel) def printDocs(ns, parent, indentLevel=0): docs = getDocFromNode(ns, parent) if docs is not None: - print(_indent(indentLevel) + "\n") + print(_indent(indentLevel) + '\n') for line in docs.splitlines(): print(_indent(indentLevel) + line) - print(_indent(indentLevel) + "\n") + print(_indent(indentLevel) + '\n') def getDocFromNode(ns, node, retval=None): - annotation_node = node.find("xs:annotation", ns) + annotation_node = node.find('xs:annotation', ns) if annotation_node is None: return retval - documentation_node = annotation_node.find("xs:documentation", ns) + documentation_node = annotation_node.find('xs:documentation', ns) if documentation_node is None: return retval - + # Be sure to grab _all_ content in the node. # In the documentation nodes, use XML entities ("<"" instead of "<") # for documentation characters that would otherwise be considered as XML. s = lxml.etree.tostring(documentation_node, method="text", pretty_print=True) - rst = s.decode().lstrip("\n") # remove any leading blank lines + rst = s.decode().lstrip('\n') # remove any leading blank lines rst = rst.rstrip() # remove any trailing white space text = textwrap.dedent(rst) # remove common leading space # substitute HTML entities in markup: "<" for "<" # thanks: http://stackoverflow.com/questions/2087370/decode-html-entities-in-python-string - try: # see #661 + try: # see #661 import html - text = html.unescape(text) except (ImportError, AttributeError): from html import parser as HTMLParser - htmlparser = HTMLParser.HTMLParser() text = htmlparser.unescape(text) @@ -398,7 +383,7 @@ def getDocFromNode(ns, node, retval=None): def addFigure(name, indentLevel=0): - fmt = """ + fmt = ''' .. compound:: .. _%s: @@ -416,23 +401,16 @@ def addFigure(name, indentLevel=0): as button in top toolbar). Set the name: "nxdl_%s.png" and move the file into the correct location using your operating system's commands. Commit the revision to version control. - """ - imageFile = "img/nxdl/nxdl_%s.png" % name - figure_id = "fig.nxdl_%s" % name + ''' + imageFile = 'img/nxdl/nxdl_%s.png' % name + figure_id = 'fig.nxdl_%s' % name if not os.path.exists(os.path.abspath(imageFile)): return - text = fmt % ( - figure_id, - imageFile, - name, - "80%", - name, - name, - ) + text = fmt % (figure_id, imageFile, name, '80%', name, name, ) indent = _indent(indentLevel) for line in text.splitlines(): print(indent + line) - print("\n") + print('\n') def pickNodesFromXpath(ns, parent, path): @@ -445,35 +423,36 @@ def main(tree, ns): for name in sorted(ELEMENT_DICT): print("") - print(".. index:: ! %s (NXDL element)\n" % name) - print(".. _%s:\n" % name) + print('.. index:: ! %s (NXDL element)\n' % name) + print('.. _%s:\n' % name) printTitle(name, indentLevel=0) - print("\n") + print('\n') print(ELEMENT_DICT[name]) - print("\n") + print('\n') addFigure(name, indentLevel=0) + print(DATATYPE_PREAMBLE) path_list = ( - "/xs:schema/xs:complexType[@name='attributeType']", - "/xs:schema/xs:element[@name='definition']", - "/xs:schema/xs:complexType[@name='definitionType']", - "/xs:schema/xs:simpleType[@name='definitionTypeAttr']", - "/xs:schema/xs:complexType[@name='dimensionsType']", - "/xs:schema/xs:complexType[@name='docType']", - "/xs:schema/xs:complexType[@name='enumerationType']", - "/xs:schema/xs:complexType[@name='fieldType']", - "/xs:schema/xs:complexType[@name='choiceType']", - "/xs:schema/xs:complexType[@name='groupType']", - "/xs:schema/xs:complexType[@name='linkType']", - "/xs:schema/xs:complexType[@name='symbolsType']", - "/xs:schema/xs:complexType[@name='basicComponent']", - "/xs:schema/xs:simpleType[@name='validItemName']", - "/xs:schema/xs:simpleType[@name='validNXClassName']", - "/xs:schema/xs:simpleType[@name='validTargetName']", - "/xs:schema/xs:simpleType[@name='nonNegativeUnbounded']", - ) + "/xs:schema/xs:complexType[@name='attributeType']", + "/xs:schema/xs:element[@name='definition']", + "/xs:schema/xs:complexType[@name='definitionType']", + "/xs:schema/xs:simpleType[@name='definitionTypeAttr']", + "/xs:schema/xs:complexType[@name='dimensionsType']", + "/xs:schema/xs:complexType[@name='docType']", + "/xs:schema/xs:complexType[@name='enumerationType']", + "/xs:schema/xs:complexType[@name='fieldType']", + "/xs:schema/xs:complexType[@name='choiceType']", + "/xs:schema/xs:complexType[@name='groupType']", + "/xs:schema/xs:complexType[@name='linkType']", + "/xs:schema/xs:complexType[@name='symbolsType']", + "/xs:schema/xs:complexType[@name='basicComponent']", + "/xs:schema/xs:simpleType[@name='validItemName']", + "/xs:schema/xs:simpleType[@name='validNXClassName']", + "/xs:schema/xs:simpleType[@name='validTargetName']", + "/xs:schema/xs:simpleType[@name='nonNegativeUnbounded']", + ) for path in path_list: nodes = pickNodesFromXpath(ns, tree, path) print("\n.. Xpath = %s\n" % path) @@ -482,12 +461,12 @@ def main(tree, ns): print(DATATYPE_POSTAMBLE) -if __name__ == "__main__": +if __name__ == '__main__': developermode = True developermode = False if developermode and len(sys.argv) != 2: path = os.path.dirname(__file__) - NXDL_SCHEMA_FILE = os.path.join(path, "..", "nxdl.xsd") + NXDL_SCHEMA_FILE = os.path.join(path, '..', 'nxdl.xsd') else: if len(sys.argv) != 2: print("usage: %s nxdl.xsd" % sys.argv[0]) @@ -496,18 +475,18 @@ def main(tree, ns): if not os.path.exists(NXDL_SCHEMA_FILE): print("Cannot find %s" % NXDL_SCHEMA_FILE) exit() - + tree = lxml.etree.parse(NXDL_SCHEMA_FILE) - NAMESPACE = "http://www.w3.org/2001/XMLSchema" - ns = {"xs": NAMESPACE} - + NAMESPACE = 'http://www.w3.org/2001/XMLSchema' + ns = {'xs': NAMESPACE} + main(tree, ns) # NeXus - Neutron and X-ray Common Data Format -# +# # Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# +# # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either diff --git a/definitions/utils/nxdl_summary.py b/definitions/utils/nxdl_summary.py index 582e501b0..905cc7fce 100755 --- a/definitions/utils/nxdl_summary.py +++ b/definitions/utils/nxdl_summary.py @@ -1,26 +1,26 @@ #!/usr/bin/env python -""" +''' Summarize the NXDL classes definitions for the given NXDL section. Re-write the index.rst file with a list of: class summary (and a hidden toctree) -""" +''' import os, sys import lxml.etree -TITLE_MARKERS = "- + ~ ^ * @".split() # used for underscoring section titles -INDENTATION = " " * 4 +TITLE_MARKERS = '- + ~ ^ * @'.split() # used for underscoring section titles +INDENTATION = ' '*4 -NAMESPACE = "http://definition.nexusformat.org/nxdl/3.1" -NS = {"nx": NAMESPACE} +NAMESPACE = 'http://definition.nexusformat.org/nxdl/3.1' +NS = {'nx': NAMESPACE} PREAMBLES = { - "base_classes": """ + 'base_classes': ''' .. index:: ! see: class definitions; base class ! base class @@ -35,9 +35,11 @@ *might* be used in an instance of that class. Consider the base classes as a set of *components* that are used to construct a data file. - """, + ''', + # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "applications": """ + + 'applications': ''' .. index:: ! see: class definitions; application definition ! application definition @@ -63,9 +65,11 @@ In application definitions involving raw data, write the raw data in the :ref:`NXinstrument` tree and then link to it from the location(s) defined in the relevant application definition. - """, + ''', + # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "contributed_definitions": """ + + 'contributed_definitions': ''' .. index:: ! see: class definitions; contributed definition ! contributed definition @@ -83,81 +87,75 @@ case not for general use. The :ref:`NIAC` is charged to review any new contributed definitions and provide feedback to the authors before ratification and acceptance as either a base class or application definition. - """, + ''', } def getSummary(nxdl_file): - """ + ''' get the summary line from each NXDL definition doc - + That's the first physical line of the doc string. - """ + ''' tree = lxml.etree.parse(nxdl_file) root = tree.getroot() - nodes = root.xpath("nx:doc", namespaces=NS) + nodes = root.xpath('nx:doc', namespaces=NS) if len(nodes) != 1: - raise RuntimeError("wrong number of nodes in NXDL: " + nxdl_file) + raise RuntimeError('wrong number of nodes in NXDL: ' + nxdl_file) text = nodes[0].text return text.strip().splitlines()[0] def command_args(): - """get the command-line arguments, handle syntax errors""" + '''get the command-line arguments, handle syntax errors''' import argparse - doc = __doc__.strip().splitlines()[0] parser = argparse.ArgumentParser(prog=sys.argv[0], description=doc) - parser.add_argument( - "section", action="store", help="NXDL section (such as *base_classes*)" - ) + parser.add_argument('section', + action='store', + help="NXDL section (such as *base_classes*)") return parser.parse_args() def main(section): if section not in PREAMBLES.keys(): - raise KeyError("unknown NXDL section: " + section) + raise KeyError('unknown NXDL section: ' + section) base_path = os.path.abspath(os.path.dirname(__file__)) - nxdl_path = os.path.abspath(os.path.join(base_path, "..", section)) + nxdl_path = os.path.abspath(os.path.join(base_path, '..', section)) if not os.path.exists(nxdl_path): - raise IOError("not found: " + nxdl_path) + raise IOError('not found: ' + nxdl_path) - rst_path = os.path.abspath( - os.path.join(base_path, "..", "manual", "source", "classes", section) - ) + rst_path = os.path.abspath(os.path.join(base_path, '..', 'manual', 'source', 'classes', section)) if not os.path.exists(rst_path): - raise IOError("not found: " + rst_path) + raise IOError('not found: ' + rst_path) - index_file = os.path.join(rst_path, "index.rst") + index_file = os.path.join(rst_path, 'index.rst') classes = [] text = [] - text.append( - """ + text.append(''' .. do NOT edit this file - automatically generated by script """ - + __file__ - ) - text.append("") + automatically generated by script ''' + __file__) + text.append('') text.append(PREAMBLES[section]) for fname in sorted(os.listdir(nxdl_path)): - if fname.endswith(".nxdl.xml"): - class_name = fname.split(".")[0] + if fname.endswith('.nxdl.xml'): + class_name = fname.split('.')[0] classes.append(class_name) summary = getSummary(os.path.join(nxdl_path, fname)) - text.append("") - text.append(":ref:`" + class_name + "`") + text.append('') + text.append(':ref:`' + class_name + '`') text.append(INDENTATION + summary) - text.append("") - text.append(".. toctree::") - text.append(INDENTATION + ":hidden:") - text.append("") + text.append('') + text.append('.. toctree::') + text.append(INDENTATION + ':hidden:') + text.append('') for cname in sorted(classes): text.append(INDENTATION + cname) - text.append("") - open(index_file, "w").writelines("\n".join(text)) + text.append('') + open(index_file, 'w').writelines('\n'.join(text)) -if __name__ == "__main__": +if __name__ == '__main__': cli = command_args() main(cli.section) diff --git a/definitions/utils/test_nxdl.py b/definitions/utils/test_nxdl.py index e7152c794..8e4544113 100644 --- a/definitions/utils/test_nxdl.py +++ b/definitions/utils/test_nxdl.py @@ -1,37 +1,33 @@ #!/bin/env python -""" +''' unit testing of NeXus definitions NXDL files and XML Schema -""" +''' import os import sys import unittest import lxml.etree -# xmllint --noout --schema nxdl.xsd base_classes/NXentry.nxdl.xml +# xmllint --noout --schema nxdl.xsd base_classes/NXentry.nxdl.xml # base_classes/NXentry.nxdl.xml validates -BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -NXDL_XSD_SCHEMA = "nxdl.xsd" +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +NXDL_XSD_SCHEMA = 'nxdl.xsd' NXDL_SCHEMA = lxml.etree.XMLSchema( - lxml.etree.parse(os.path.join(BASE_DIR, NXDL_XSD_SCHEMA)) -) + lxml.etree.parse( + os.path.join(BASE_DIR, NXDL_XSD_SCHEMA))) -NXDL_CATEGORY_NAMES = "base_classes applications contributed_definitions".split() +NXDL_CATEGORY_NAMES = 'base_classes applications contributed_definitions'.split() -class NXDL_Invalid(Exception): - pass - - -class NXDL_Valid(Exception): - pass +class NXDL_Invalid(Exception): pass +class NXDL_Valid(Exception): pass def isNXDL(fname): - return fname.endswith(".nxdl.xml") + return fname.endswith('.nxdl.xml') def get_NXDL_file_list(): @@ -45,73 +41,73 @@ def get_NXDL_file_list(): def validate_xml(xml_file_name): - """ + ''' validate an NXDL XML file against an XML Schema file :param str xml_file_name: name of XML file - """ + ''' try: xml_tree = lxml.etree.parse(xml_file_name) except lxml.etree.XMLSyntaxError as exc: - msg = xml_file_name + " : " + str(exc) + msg = xml_file_name + ' : ' + str(exc) raise NXDL_Invalid(msg) try: result = NXDL_SCHEMA.assertValid(xml_tree) # there is no assertNotRaises so raise this when successful raise NXDL_Valid except lxml.etree.DocumentInvalid as exc: - msg = xml_file_name + " : " + str(exc) + msg = xml_file_name + ' : ' + str(exc) raise NXDL_Invalid(msg) class TestMaker(type): + def __new__(cls, clsname, bases, dct): - # Add a method to the class' __dict__ for every + # Add a method to the class' __dict__ for every # file name in the NXDL file list. - cat_number_dict = {c: str(i + 1) for i, c in enumerate(NXDL_CATEGORY_NAMES)} + cat_number_dict = {c: str(i+1) for i, c in enumerate(NXDL_CATEGORY_NAMES)} for fname in get_NXDL_file_list(): category, nxdl_name = os.path.split(fname) category_number = cat_number_dict[category] point = nxdl_name.find(".") nxdl_name = nxdl_name[:point] - test_name = "test" + test_name = 'test' # since these will be sorted, get the categories in the desired order - test_name += "__" + str(category_number) - test_name += "__" + category - test_name += "__" + nxdl_name + test_name += '__' + str(category_number) + test_name += '__' + category + test_name += '__' + nxdl_name dct[test_name] = cls.make_test(fname) return super(TestMaker, cls).__new__(cls, clsname, bases, dct) @staticmethod def make_test(nxdl_file_name): + def test_wrap(self): # test body for each NXDL file test with self.assertRaises(NXDL_Valid): validate_xml(nxdl_file_name) self.assertRaises(NXDL_Valid, validate_xml, nxdl_file_name) - return test_wrap - class Individual_NXDL_Tests(unittest.TestCase, metaclass=TestMaker): - """ + ''' run all tests created in TestMaker() class, called by suite() - """ + ''' def suite(*args, **kw): - """gather all the tests together in a suite, called by run()""" + '''gather all the tests together in a suite, called by run()''' test_suite = unittest.TestSuite() test_suite.addTests(unittest.makeSuite(Individual_NXDL_Tests)) return test_suite def run(): - """run all the unit tests""" - runner = unittest.TextTestRunner(verbosity=2) + '''run all the unit tests''' + runner=unittest.TextTestRunner(verbosity=2) runner.run(suite()) -if __name__ == "__main__": +if __name__ == '__main__': run() diff --git a/definitions/utils/test_nxdl2rst.py b/definitions/utils/test_nxdl2rst.py index 733e3f752..e72b9fdac 100644 --- a/definitions/utils/test_nxdl2rst.py +++ b/definitions/utils/test_nxdl2rst.py @@ -1,8 +1,8 @@ #!/bin/env python -""" +''' unit testing: NXDL to RST for documentation -""" +''' import os import sys @@ -14,49 +14,47 @@ class Capture_stdout(list): - """ + ''' capture all printed output (to stdout) into list - + # http://stackoverflow.com/questions/16571150/how-to-capture-stdout-output-from-a-python-function-call - """ - + ''' def __enter__(self): self._stdout = sys.stdout sys.stdout = self._stringio = StringIO() return self - def __exit__(self, *args): self.extend(self._stringio.getvalue().splitlines()) - del self._stringio # free up some memory + del self._stringio # free up some memory sys.stdout = self._stdout class Issue_524_Clarify_Optional_or_Required(unittest.TestCase): - """ + ''' make it obvious what is required and what is optional - + **field**: (optional or required) NX_TYPE - """ - + ''' + def test_base_class_NXentry(self): - expected_lines = """ + expected_lines = ''' **definition**: (optional) :ref:`NX_CHAR ` **DATA**: (optional) :ref:`NXdata` **notes**: (optional) :ref:`NXnote` **@default**: (optional) :ref:`NX_CHAR ` - """.strip().splitlines() - - self.apply_tests("base_classes", "NXentry", expected_lines) - + '''.strip().splitlines() + + self.apply_tests('base_classes', 'NXentry', expected_lines) + def test_base_class_NXuser(self): - expected_lines = """ + expected_lines = ''' **name**: (optional) :ref:`NX_CHAR ` - """.strip().splitlines() - - self.apply_tests("base_classes", "NXuser", expected_lines) - + '''.strip().splitlines() + + self.apply_tests('base_classes', 'NXuser', expected_lines) + def test_application_definition_NXcanSAS(self): - expected_lines = """ + expected_lines = ''' **definition**: (required) :ref:`NX_CHAR ` **title**: (required) :ref:`NX_CHAR ` **run**: (required) :ref:`NX_CHAR ` @@ -78,20 +76,18 @@ def test_application_definition_NXcanSAS(self): **@canSAS_class**: (required) :ref:`NX_CHAR ` **@signal**: (required) :ref:`NX_CHAR ` **@I_axes**: (required) :ref:`NX_CHAR ` - """.strip().splitlines() - - self.apply_tests("applications", "NXcanSAS", expected_lines) + '''.strip().splitlines() + + self.apply_tests('applications', 'NXcanSAS', expected_lines) def apply_tests(self, category, class_name, expected_lines): - nxdl_file = os.path.join( - os.path.dirname(__file__), "..", category, class_name + ".nxdl.xml" - ) + nxdl_file = os.path.join(os.path.dirname(__file__),'..', category, class_name+'.nxdl.xml') self.assertTrue(os.path.exists(nxdl_file), nxdl_file) - - sys.argv.insert(0, "python") + + sys.argv.insert(0, 'python') with Capture_stdout() as printed_lines: nxdl2rst.print_rst_from_nxdl(nxdl_file) - + printed_lines = [_.strip() for _ in printed_lines] for line in expected_lines: expected = line.strip() @@ -99,7 +95,7 @@ def apply_tests(self, category, class_name, expected_lines): def suite(*args, **kw): - """gather all the tests together in a suite, called by run()""" + '''gather all the tests together in a suite, called by run()''' test_suite = unittest.TestSuite() test_suite_list = [ Issue_524_Clarify_Optional_or_Required, @@ -110,10 +106,10 @@ def suite(*args, **kw): def run(): - """run all the unit tests""" - runner = unittest.TextTestRunner(verbosity=2) + '''run all the unit tests''' + runner=unittest.TextTestRunner(verbosity=2) runner.run(suite()) -if __name__ == "__main__": +if __name__ == '__main__': run() diff --git a/definitions/utils/test_suite.py b/definitions/utils/test_suite.py index 92b8bb246..11c646990 100644 --- a/definitions/utils/test_suite.py +++ b/definitions/utils/test_suite.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -""" +''' unit testing of the NeXus definitions -""" +''' import os import unittest @@ -12,21 +12,20 @@ def suite(*args, **kw): import test_nxdl import test_nxdl2rst - test_suite = unittest.TestSuite() test_list = [ test_nxdl, test_nxdl2rst, - ] + ] for test in test_list: test_suite.addTest(test.suite()) return test_suite -if __name__ == "__main__": +if __name__ == '__main__': owd = os.getcwd() - runner = unittest.TextTestRunner(verbosity=2) + runner=unittest.TextTestRunner(verbosity=2) result = runner.run(suite()) os.chdir(owd) sys.exit(len(result.errors)) diff --git a/definitions/utils/types2rst.py b/definitions/utils/types2rst.py index 2faa1a62f..d724438ac 100755 --- a/definitions/utils/types2rst.py +++ b/definitions/utils/types2rst.py @@ -1,24 +1,24 @@ #!/usr/bin/env python -""" +''' Read the the NeXus NXDL types specification and find all the valid data types. Write a restructured text (.rst) document for use in the NeXus manual in the NXDL chapter. -""" +''' import units2rst -if __name__ == "__main__": - units2rst.worker("primitiveType", section="data") +if __name__ == '__main__': + units2rst.worker('primitiveType', section = 'data') # NeXus - Neutron and X-ray Common Data Format -# +# # Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# +# # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either diff --git a/definitions/utils/units2rst.py b/definitions/utils/units2rst.py index b3ac8e4e7..bce2cfe64 100755 --- a/definitions/utils/units2rst.py +++ b/definitions/utils/units2rst.py @@ -1,18 +1,18 @@ #!/usr/bin/env python -""" +''' Read the the NeXus NXDL types specification and find all the valid types of units. Write a restructured text (.rst) document for use in the NeXus manual in the NXDL chapter. -""" +''' import os, sys import lxml.etree -def worker(nodeMatchString, section="units"): +def worker(nodeMatchString, section = 'units'): if len(sys.argv) != 2: print("usage: %s nxdlTypes.xsd" % sys.argv[0]) exit() @@ -20,72 +20,72 @@ def worker(nodeMatchString, section="units"): if not os.path.exists(NXDL_TYPES_FILE): print("Cannot find %s" % NXDL_TYPES_FILE) exit() - + tree = lxml.etree.parse(NXDL_TYPES_FILE) - - output = [".. auto-generated by %s -- DO NOT EDIT" % sys.argv[0]] - output.append("") - - labels = ("term", "description") - output.append(".. nodeMatchString : %s" % nodeMatchString) - output.append("") + + output = ['.. auto-generated by %s -- DO NOT EDIT' % sys.argv[0]] + output.append('') + + labels = ('term', 'description') + output.append('.. nodeMatchString : %s' % nodeMatchString) + output.append('') db = {} - - NAMESPACE = "http://www.w3.org/2001/XMLSchema" - ns = {"xs": NAMESPACE} - root = tree.xpath("//xs:schema", namespaces=ns)[0] - s = "//xs:simpleType" + + NAMESPACE = 'http://www.w3.org/2001/XMLSchema' + ns = {'xs': NAMESPACE} + root = tree.xpath('//xs:schema', namespaces=ns)[0] + s = '//xs:simpleType' node_list = tree.xpath("//xs:simpleType", namespaces=ns) - + # get the names of all the types of units members = [] for node in node_list: - if node.get("name") == nodeMatchString: - union = node.xpath("xs:union", namespaces=ns) - members = union[0].get("memberTypes", "").split() - + if node.get('name') == nodeMatchString: + union = node.xpath('xs:union', namespaces=ns) + members = union[0].get('memberTypes', '').split() + # get the definition of each type of units for node in node_list: - node_name = node.get("name") - if "nxdl:" + node_name in members: - words = node.xpath("xs:annotation/xs:documentation", namespaces=ns)[0] + node_name = node.get('name') + if 'nxdl:' + node_name in members: + words = node.xpath('xs:annotation/xs:documentation', namespaces=ns)[0] examples = [] for example in words.iterchildren(): nm = example.attrib.get("name") if nm is not None and nm == "example": - examples.append("``" + example.text + "``") + examples.append("``"+example.text+"``") a = words.text if len(examples) > 0: a = a.strip() + ", example(s): " + " | ".join(examples) db[node_name] = a - # for item in node.xpath('xs:restriction//xs:enumeration', namespaces=ns): - # key = '%s' % item.get('value') - # words = item.xpath('xs:annotation/xs:documentation', namespaces=ns)[0] - # db[key] = words.text - - print("\n".join(output)) - +# for item in node.xpath('xs:restriction//xs:enumeration', namespaces=ns): +# key = '%s' % item.get('value') +# words = item.xpath('xs:annotation/xs:documentation', namespaces=ns)[0] +# db[key] = words.text + + print('\n'.join(output)) + # this list is too long to make this a table in latex # for two columns, a Sphinx fieldlist will do just as well for key in sorted(db): - print(".. index:: ! %s (%s type)\n" % (key, section)) # index entry - print(".. _%s:\n" % key) # cross-reference point - print(":%s:" % key) + print('.. index:: ! %s (%s type)\n' % (key, section)) # index entry + print('.. _%s:\n' % key) # cross-reference point + print(':%s:' % key) for line in db[key].splitlines(): - print(" %s" % line) - print("") + print(' %s' % line) + print('') -if __name__ == "__main__": - # sys.argv.append('../nxdlTypes.xsd') # FIXME: developer only -- remove for production!!! - worker("anyUnitsAttr") +if __name__ == '__main__': + #sys.argv.append('../nxdlTypes.xsd') # FIXME: developer only -- remove for production!!! + worker('anyUnitsAttr') # NeXus - Neutron and X-ray Common Data Format -# +# # Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# +# # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either diff --git a/definitions/utils/update_copyright_date.py b/definitions/utils/update_copyright_date.py index f62229ca4..f81b72f18 100755 --- a/definitions/utils/update_copyright_date.py +++ b/definitions/utils/update_copyright_date.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -""" +''' update the copyright date in all NeXus text files This is the bash command to find all matching lines:: @@ -8,7 +8,7 @@ grep -iR copyright | grep -i "(c)" | grep -i nexus See copyright text at bottom of this file for example. -""" +''' import os, sys import mimetypes @@ -17,20 +17,19 @@ import datetime YEAR = datetime.datetime.now().year -LEFT_SIDE_TEXT_MATCH = "Copyright (C) " -RIGHT_SIDE_TEXT_MATCH = " NeXus International Advisory Committee (NIAC)" +LEFT_SIDE_TEXT_MATCH = 'Copyright (C) ' +RIGHT_SIDE_TEXT_MATCH = ' NeXus International Advisory Committee (NIAC)' def update(filename): - """update the copyright year in the file""" - + '''update the copyright year in the file''' def position(line, key): pos = line.find(key) if pos >= 0: - if line.find("NIAC") >= 0: + if line.find('NIAC') >= 0: return pos return None - + if not os.path.exists(filename): return changes = [] @@ -38,7 +37,7 @@ def position(line, key): for number, line in enumerate(buf): pos = position(line, LEFT_SIDE_TEXT_MATCH) if pos is None: - continue # no match + continue # no match pos += len(LEFT_SIDE_TEXT_MATCH) text_l = line[:pos] @@ -48,65 +47,65 @@ def position(line, key): text_r = text[pos:] try: - years = list(map(int, text[:pos].split("-"))) + years = list(map(int, text[:pos].split('-'))) if len(years) in (1, 2): if len(years) == 1: years.append(YEAR) elif len(years) == 2: years[1] = YEAR - line_new = text_l + "-".join(map(str, years)) + text_r + line_new = text_l + '-'.join(map(str, years)) + text_r changes.append(list((number, line_new))) except Exception as _exc: print(number, filename, str(_exc)) for number, line in changes: buf[number] = line if len(changes) > 0: - print("Update: ", filename) - fp = open(filename, "w") + print('Update: ', filename) + fp = open(filename, 'w') fp.writelines(buf) fp.close() def find_source_files(path): - """walk the source_path directories accumulating files to be checked""" + '''walk the source_path directories accumulating files to be checked''' file_list = [] for root, dirs, files in os.walk(path): - if root.find("/.git") < 0 or root.find("/kits") < 0: + if root.find('/.git') < 0 or root.find('/kits') < 0: file_list = file_list + [os.path.join(root, _) for _ in files] return file_list def sift_file_list(file_list): - """remove known non-text files and paths""" + '''remove known non-text files and paths''' new_list = [] - acceptable_mime_types = """ + acceptable_mime_types = ''' application/xml application/x-msdos-program application/xslt+xml - """.strip().split() - ignore_extensions = """ + '''.strip().split() + ignore_extensions = ''' .dia .vsdx .h5 .nx .hdf5 .hdf .nx5 .pyc - """.strip().split() + '''.strip().split() for fn in file_list: _fn = os.path.split(fn)[-1] mime = mimetypes.guess_type(fn)[0] - if fn.find("/.git") >= 0: + if fn.find('/.git') >= 0: continue - if fn.find("/.settings") >= 0: + if fn.find('/.settings') >= 0: continue - if fn.find("/kits") >= 0: + if fn.find('/kits') >= 0: continue - if fn.find("/build") >= 0: + if fn.find('/build') >= 0: continue if os.path.splitext(fn)[-1] in ignore_extensions: continue - if mime is None or mime.startswith("text/") or mime in acceptable_mime_types: + if mime is None or mime.startswith('text/') or mime in acceptable_mime_types: new_list.append(fn) return new_list def is_definitions_directory(basedir): - """test if ``basedir`` is a NeXus definitions directory""" + '''test if ``basedir`` is a NeXus definitions directory''' # look for the expected files and subdirectories in the root directory for item_list in ROOT_DIR_EXPECTED_RESOURCES.values(): for item in item_list: @@ -116,63 +115,61 @@ def is_definitions_directory(basedir): def qualify_inputs(root_dir): - """raise error if this program cannot continue, based on the inputs""" + '''raise error if this program cannot continue, based on the inputs''' if not os.path.exists(root_dir): - raise RuntimeError("Cannot find " + root_dir) + raise RuntimeError('Cannot find ' + root_dir) if not os.path.isdir(root_dir): - raise RuntimeError("Not a directory: " + root_dir) + raise RuntimeError('Not a directory: ' + root_dir) if not is_definitions_directory(root_dir): - msg = "Not a NeXus definitions root directory " + root_dir + msg = 'Not a NeXus definitions root directory ' + root_dir raise RuntimeError(msg) def command_args(): - """get the command-line arguments, handle syntax errors""" + '''get the command-line arguments, handle syntax errors''' import argparse - doc = __doc__.strip().splitlines()[0] parser = argparse.ArgumentParser(prog=sys.argv[0], description=doc) - parser.add_argument( - "defs_dir", action="store", help="NeXus definitions root directory" - ) + parser.add_argument('defs_dir', + action='store', + help="NeXus definitions root directory") return parser.parse_args() def main(): - """ + ''' standard command-line processing - + source directory (NeXus definitions dir) named as command line argument target directory is specified (or defaults to present working directory) - """ + ''' cli = command_args() root_dir = os.path.abspath(cli.defs_dir) qualify_inputs(root_dir) - + file_list = sift_file_list(find_source_files(root_dir)) for fn in file_list: update(fn) def __developer_build_setup__(): - """for use with source-code debugger ONLY""" + '''for use with source-code debugger ONLY''' import shutil - # sys.argv.append('-h') - sys.argv.append("..") + sys.argv.append('..') -if __name__ == "__main__": - # __developer_build_setup__() +if __name__ == '__main__': + #__developer_build_setup__() main() # NeXus - Neutron and X-ray Common Data Format -# +# # Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# +# # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either