Skip to content

Commit

Permalink
Apply black #578
Browse files Browse the repository at this point in the history
  • Loading branch information
@nesnoj
nesnoj committed Oct 11, 2024
1 parent 8bde6af commit c93b8f4
Show file tree
Hide file tree
Showing 15 changed files with 399 additions and 231 deletions.
33 changes: 17 additions & 16 deletions docs/conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,15 @@
#
import os
import sys
sys.path.insert(0, os.path.abspath('../open_mastr'))

sys.path.insert(0, os.path.abspath("../open_mastr"))


# -- Project information -----------------------------------------------------

project = 'open-MaStR'
copyright = '2022 Reiner Lemoine Institut and fortiss'
author = ''
project = "open-MaStR"
copyright = "2022 Reiner Lemoine Institut and fortiss"
author = ""


# -- General configuration ---------------------------------------------------
Expand All @@ -28,36 +29,36 @@
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autosectionlabel',
'sphinx.ext.autodoc',
'sphinx.ext.napoleon',
'sphinx_tabs.tabs',
'm2r2',
"sphinx.ext.autosectionlabel",
"sphinx.ext.autodoc",
"sphinx.ext.napoleon",
"sphinx_tabs.tabs",
"m2r2",
]

source_suffix = [".rst", ".md"]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]


# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'
html_theme = "sphinx_rtd_theme"

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_css_files = ['custom.css']
html_static_path = ["_static"]
html_css_files = ["custom.css"]

# Autodoc config
autoclass_content = 'both'
# Autodoc config
autoclass_content = "both"
70 changes: 46 additions & 24 deletions open_mastr/soap_api/metadata/description.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,19 @@ def __init__(self, xml=None):
self.xml = fh.read()
else:
# If no XML file is given, the file is read from an URL
zipurl = 'https://www.marktstammdatenregister.de/MaStRHilfe/files/' \
'webdienst/Dienstbeschreibung_1_2_39_Produktion.zip'
zipurl = (
"https://www.marktstammdatenregister.de/MaStRHilfe/files/"
"webdienst/Dienstbeschreibung_1_2_39_Produktion.zip"
)

with urlopen(zipurl) as zipresp:
with ZipFile(BytesIO(zipresp.read())) as zfile:
self.xml = zfile.read('xsd/mastrbasetypes.xsd')


self.xml = zfile.read("xsd/mastrbasetypes.xsd")

# Parse XML and extract relevant data
parsed = xmltodict.parse(self.xml, process_namespaces=False)
self.complex_types = parsed['schema']["complexType"]
self.simple_types = parsed['schema']["simpleType"]
self.complex_types = parsed["schema"]["complexType"]
self.simple_types = parsed["schema"]["simpleType"]

# Prepare parsed data for documentational purposes
abstract_types, parameters, responses, types = self._filter_type_descriptions()
Expand Down Expand Up @@ -78,13 +78,17 @@ def _filter_type_descriptions(self):
raise ValueError("Ohh...")
else:
# Filter all functions
if item["@name"].startswith(("Get", "Set", "Erneute", "Verschiebe", "Delete")):
if item["@name"].startswith(
("Get", "Set", "Erneute", "Verschiebe", "Delete")
):
functions.append(item)

# Further split the list of functions into paramters and responses
if item["@name"].endswith("Parameter"):
if "complexContent" in item.keys():
parameters[item["@name"]] = item["complexContent"]["extension"]
parameters[item["@name"]] = item["complexContent"][
"extension"
]
else:
parameters[item["@name"]] = item
elif item["@name"].endswith("Antwort"):
Expand All @@ -111,12 +115,14 @@ def prepare_simple_type(self):

for simple_type in self.simple_types:
if "enumeration" in simple_type["restriction"]:
possible_values = [_["@value"] for _ in simple_type["restriction"]["enumeration"]]
possible_values = [
_["@value"] for _ in simple_type["restriction"]["enumeration"]
]
else:
possible_values = []
simple_types_doc[simple_type["@name"]] = {
"type": simple_type["restriction"]["@base"],
"values": possible_values
"values": possible_values,
}
return simple_types_doc

Expand All @@ -140,49 +146,61 @@ def functions_data_documentation(self):
if "annotation" in fcn["sequence"]["element"]:
fcn_data = [fcn["sequence"]["element"]]
else:
fcn_data = self.types[fcn["sequence"]["element"]["@type"].split(":")[1]]["sequence"]["element"]
fcn_data = self.types[
fcn["sequence"]["element"]["@type"].split(":")[1]
]["sequence"]["element"]
else:
print(type(fcn["sequence"]))
print(fcn["sequence"])
raise ValueError

# Add data for inherited columns from base types
if "@base" in fcn:
if not fcn["@base"] == 'mastr:AntwortBasis':
fcn_data = _collect_columns_of_base_type(self.types, fcn["@base"].split(":")[1], fcn_data)
if not fcn["@base"] == "mastr:AntwortBasis":
fcn_data = _collect_columns_of_base_type(
self.types, fcn["@base"].split(":")[1], fcn_data
)
function_docs[fcn_name] = {}
for column in fcn_data:
# Replace MaStR internal types with more general ones
if column["@type"].startswith("mastr:"):
try:
column_type = self.simple_types_prepared[column["@type"].split(":")[1]]["type"]
column_type = self.simple_types_prepared[
column["@type"].split(":")[1]
]["type"]
except KeyError:
column_type = column["@type"]
else:
column_type = column["@type"]

if "annotation" in column.keys():
description = column["annotation"]["documentation"].get("#text", None)
description = column["annotation"]["documentation"].get(
"#text", None
)
if description:
description = re.sub(" +", " ", description.replace("\n", ""))
description = re.sub(
" +", " ", description.replace("\n", "")
)
function_docs[fcn_name][column["@name"]] = {
"type": column_type,
"description": description,
"example": column["annotation"]["documentation"].get("m-ex", None)
"type": column_type,
"description": description,
"example": column["annotation"]["documentation"].get(
"m-ex", None
),
}
else:
function_docs[fcn_name][column["@name"]] = {
"type": column_type,
# TODO: insert information from simple type here
"description": None,
"example": None
"example": None,
}

# Hack in a descrition for a column that gets created after download while flattening data
function_docs["GetEinheitWind"]["HerstellerId"] = {
"type": "str",
"description": "Id des Herstellers der Einheit",
"example": 923
"example": 923,
}

return function_docs
Expand All @@ -193,7 +211,11 @@ def _collect_columns_of_base_type(base_types, base_type_name, fcn_data):
fcn_data += type_description["extension"]["sequence"]["element"]

if "@base" in type_description["extension"]:
if not type_description["extension"]["@base"] == 'mastr:AntwortBasis':
fcn_data = _collect_columns_of_base_type(base_types, type_description["extension"]["@base"].split(":")[1], fcn_data)
if not type_description["extension"]["@base"] == "mastr:AntwortBasis":
fcn_data = _collect_columns_of_base_type(
base_types,
type_description["extension"]["@base"].split(":")[1],
fcn_data,
)

return fcn_data
24 changes: 15 additions & 9 deletions open_mastr/utils/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
# -*- coding: utf-8 -*-



"""
Service functions for logging
Expand All @@ -26,7 +25,11 @@

import logging
import logging.config
from open_mastr.utils.constants import TECHNOLOGIES, API_LOCATION_TYPES, ADDITIONAL_TABLES
from open_mastr.utils.constants import (
TECHNOLOGIES,
API_LOCATION_TYPES,
ADDITIONAL_TABLES,
)


log = logging.getLogger(__name__)
Expand Down Expand Up @@ -57,7 +60,7 @@ def get_output_dir():
"""

if "OUTPUT_PATH" in os.environ:
return os.environ.get('OUTPUT_PATH')
return os.environ.get("OUTPUT_PATH")

return get_project_home_dir()

Expand All @@ -76,7 +79,7 @@ def get_data_version_dir():
data_version = get_data_config()

if "OUTPUT_PATH" in os.environ:
return os.path.join(os.environ.get('OUTPUT_PATH'), "data", data_version)
return os.path.join(os.environ.get("OUTPUT_PATH"), "data", data_version)

return os.path.join(get_project_home_dir(), "data", data_version)

Expand Down Expand Up @@ -230,18 +233,21 @@ def _filenames_generator():
}

# Add file names of processed data
filenames["postprocessed"] = {
tech: f"{prefix}_{tech}.csv" for tech in TECHNOLOGIES
}
filenames["postprocessed"] = {tech: f"{prefix}_{tech}.csv" for tech in TECHNOLOGIES}

# Add filenames for location data
filenames["raw"].update(
{loc: f"{prefix}_{loc}_raw.csv" for loc in API_LOCATION_TYPES}
)

# Add filenames for additional tables
filenames["raw"].update({"additional_table":
{addit_table: f"{prefix}_{addit_table}_raw.csv" for addit_table in ADDITIONAL_TABLES}}
filenames["raw"].update(
{
"additional_table": {
addit_table: f"{prefix}_{addit_table}_raw.csv"
for addit_table in ADDITIONAL_TABLES
}
}
)

# Add metadata file
Expand Down
Loading

0 comments on commit c93b8f4

Please sign in to comment.