refactor: python doc script classes (#301)

* feat: adding snapshot tests * allow snapshot tests to fail * more tests on ugly classes * refactor away the document classes migrate gen_topology_groupmethods.py migrate gen_topologyattr_defaults.py migrate FormatOverview migrate CoordinateReaders migrate gen_format_overview_classes migrate TopologyParsers migrate TopologyAttrs migrate ConnectivityAttrs remove dead code, never executed remove all class attributes all non-strict mypy checks pass, mypy in pre-commit first nice simplification another nice simplification more simplifications further cleanups further trimmings continue to refactor update base.py harmonize to use private functions simpler code architecture more refactoring mypy strict mode on some files finish mypy changes flake8 on everything Update doc/source/scripts/base.py Co-authored-by: Lily Wang <[email protected]> Update doc/source/scripts/gen_format_overview_classes.py Co-authored-by: Lily Wang <[email protected]> Update doc/source/scripts/gen_format_overview_classes.py Co-authored-by: Lily Wang <[email protected]> Update doc/source/scripts/gen_format_overview_classes.py Co-authored-by: Lily Wang <[email protected]> Update doc/source/scripts/gen_format_overview_classes.py Co-authored-by: Lily Wang <[email protected]> review comments from Lily fixing mypy * Update doc/source/scripts/base.py Co-authored-by: Lily Wang <[email protected]> * Update doc/source/scripts/gen_format_overview_classes.py Co-authored-by: Lily Wang <[email protected]> * Update doc/source/scripts/gen_topologyparser_attrs.py Co-authored-by: Lily Wang <[email protected]> * Update doc/source/scripts/gen_topologyparser_attrs.py Co-authored-by: Lily Wang <[email protected]> * Update doc/source/scripts/gen_topologyparser_attrs.py Co-authored-by: Lily Wang <[email protected]> * Update doc/source/scripts/gen_topologyparser_attrs.py Co-authored-by: Lily Wang <[email protected]> * Update doc/source/scripts/gen_topologyparser_attrs.py Co-authored-by: Lily Wang <[email protected]> * review: comments from Lily * fix tests * more flake8 goodies * run linter --------- Co-authored-by: Lily Wang <[email protected]>
MDAnalysis · Oct 9, 2023 · a952cb2 · a952cb2
1 parent 5e660d0
commit a952cb2
Show file tree

Hide file tree

Showing 21 changed files with 682 additions and 422 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,6 +2,12 @@ repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.2.0
     hooks:
+    -   id: check-added-large-files
+    -   id: check-case-conflict
+    -   id: check-executables-have-shebangs
+    -   id: check-merge-conflict
+    -   id: check-shebang-scripts-are-executable
+    -   id: check-symlinks
     -   id: check-yaml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
@@ -17,3 +23,21 @@ repos:
     hooks:
     -   id: black
         args: [--line-length=79]
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.5.1
+    hooks:
+    -   id: mypy
+        args: [--ignore-missing-imports, --install-types, --non-interactive, --strict]
+        exclude: "/tests/.*\\.py|clean_example_notebooks.py|update_json_stubs_sitemap.py"
+-   repo: https://github.com/PyCQA/flake8
+    rev: 6.1.0
+    hooks:
+    -   id: flake8
+        additional_dependencies: [
+            'flake8-simplify',
+            'flake8-comprehensions',
+            'flake8-bugbear',
+            'darglint',
+            'flake8-pep585',
+            'Flake8-pyproject',
+        ]
diff --git a/doc/source/conf.py b/doc/source/conf.py
@@ -23,7 +23,7 @@
 project = "MDAnalysis User Guide"
 
 
-def sort_authors(filename):
+def sort_authors(filename: str) -> list[str]:
     """Generate sorted list of authors from AUTHORS"""
     authors = []
     with open(filename, "r") as f:
@@ -144,7 +144,7 @@ def sort_authors(filename):
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
-html_css_files = []
+html_css_files: list[str] = []
 
 # Custom sidebar templates, maps document names to template names.
 # alabaster sidebars
@@ -170,8 +170,9 @@ def sort_authors(filename):
 }
 
 # nbsphinx
-html_js_files = [
-    # "https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js",
+html_js_files: list[str] = [
+    # 'https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js',
+    # DEFAULT_EMBED_REQUIREJS_URL,
 ]
 
 ipython_warning_is_error = False

diff --git a/doc/source/scripts/.coverage b/doc/source/scripts/.coverage
diff --git a/doc/source/scripts/base.py b/doc/source/scripts/base.py
@@ -1,15 +1,65 @@
-from __future__ import print_function
-
 import os
 import pathlib
 import sys
 import textwrap
-from collections import defaultdict
+from collections.abc import Callable, Iterable
+from typing import Any, Optional, Type
 
+import pandas as pd
 import tabulate
 
 
-class TableWriter(object):
+def _run_method(method: Callable[..., str], *args: Any) -> str:
+    val = method(*args)
+    return val
+
+
+def _generate_row(
+    *, column_spec: list[tuple[str, Callable[..., str]]], args: Iterable[Any]
+) -> dict[str, str]:
+    row = {}
+    for heading, method in column_spec:
+        val = _run_method(method, *args)
+        row[heading] = val
+    return row
+
+
+def _generate_table(
+    *,
+    input_items: Iterable[Any],
+    column_spec: list[tuple[str, Callable[..., str]]],
+) -> pd.DataFrame:
+    rows = []
+    for args in input_items:
+        if not isinstance(args, Iterable):
+            args = [args]
+        line = _generate_row(column_spec=column_spec, args=args)
+        rows.append(line)
+    df = pd.DataFrame(rows)
+    return df
+
+
+def write_table(
+    *,
+    path: str,
+    headers: list[str],
+    lines: list[list[str]],
+    include_table: Optional[str] = None,
+) -> None:
+    parent_directory = pathlib.Path(path).parent
+    parent_directory.mkdir(exist_ok=True, parents=True)
+    with open(path, "w") as f:
+        f.write(f'..\n    Generated by {sys.argv[0].split("/")[-1]}\n\n')
+        if include_table:
+            f.write(f".. table:: {include_table}\n\n")
+        tabled = tabulate.tabulate(lines, headers=headers, tablefmt="rst")
+        if include_table:
+            tabled = textwrap.indent(tabled, "    ")
+        f.write(tabled)
+    print("Wrote ", path)
+
+
+class TableWriter:
     """
     For writing tables with easy column switching.
 
@@ -18,91 +68,77 @@ class TableWriter(object):
     Filename relative to source.
     """
 
-    filename = ""
-    include_table = False
-    headings = []
-    preprocess = []
-    postprocess = []
-    sort = True
+    def __init__(
+        self,
+        column_spec: list[tuple[str, Callable[..., str]]],
+        lines: list[list[str]],
+        filename: str = "",
+        include_table: Optional[str] = None,
+        sort: bool = True,
+        input_items: Optional[Iterable[Any]] = None,
+    ):
+        if column_spec:
+            assert input_items
+        assert (column_spec and not lines) or (lines and not column_spec)
+        stem = os.getcwd().split("source")[0]
+        self.path = os.path.join(stem, "source", filename)
+        self.filename = filename
+        self.include_table = include_table
+        self.sort = sort
+        self.input_items = input_items
+        self.column_spec = column_spec
+        self.lines = lines
+        self._df = pd.DataFrame()
 
-    def __getattr__(self, key: str) -> list:
-        return self.fields[key]
+    @property
+    def headers(self) -> list[str]:
+        return [column_name for column_name, _ in self.column_spec]
 
-    def __init__(self, *args, **kwargs):
-        stem = os.getcwd().split("source")[0]
-        self.path = os.path.join(stem, "source", self.filename)
-        self.fields = defaultdict(list)
+    @property
+    def fields(self) -> pd.DataFrame:
+        return self._df
 
-        parent_directory = pathlib.Path(self.path).parent
-        parent_directory.mkdir(exist_ok=True, parents=True)
-        self.get_lines(*args, **kwargs)
-        self.write_table()
+    def generate_lines_and_write_table(self) -> None:
+        df = _generate_table(
+            input_items=self.input_items or [],
+            column_spec=self.column_spec,
+        )
 
-    def _run_method(self, method, *args, **kwargs):
-        sanitized = self.sanitize_name(method)
-        meth = getattr(self, sanitized)
-        val = meth(*args, **kwargs)
-        self.fields[method].append(val)
-        return val
-
-    @staticmethod
-    def sanitize_name(name):
-        return "_" + name.replace(" ", "_").replace("/", "_").lower()
-
-    def get_lines(self, *args, **kwargs):
-        lines = []
-        for items in self._set_up_input():
-            try:
-                lines.append(self.get_line(*items))
-            except TypeError:  # one argument
-                lines.append(self.get_line(items))
-        if self.sort:
-            lines = sorted(lines)
+        lines = df.values.tolist()
+        lines = sorted(lines) if self.sort else lines
         self.lines = lines
+        self._df = df
+        self.write_table()
 
-    def get_line(self, *args):
-        line = []
-        for p in self.preprocess:
-            self._run_method(p, *args)
-        for h in self.headings:
-            line.append(self._run_method(h, *args))
-        for p in self.postprocess:
-            self._run_method(p, *args)
-        return line
-
-    def write_table(self):
-        with open(self.path, "w") as f:
-            f.write(f'..\n    Generated by {sys.argv[0].split("/")[-1]}\n\n')
-            if self.include_table:
-                f.write(f".. table:: {self.include_table}\n\n")
-            tabled = tabulate.tabulate(
-                self.lines, headers=self.headings, tablefmt="rst"
-            )
-            if self.include_table:
-                tabled = textwrap.indent(tabled, "    ")
-            f.write(tabled)
-        print("Wrote ", self.filename)
-
-    # ==== HELPER FUNCTIONS ==== #
-
-    @staticmethod
-    def sphinx_class(klass, tilde=True):
-        prefix = "~" if tilde else ""
-        return ":class:`{}{}.{}`".format(
-            prefix, klass.__module__, klass.__name__
+    def write_table(self) -> None:
+        write_table(
+            path=self.path,
+            headers=self.headers,
+            lines=self.lines,
+            include_table=self.include_table,
         )
 
-    @staticmethod
-    def sphinx_meth(meth, tilde=True):
-        prefix = "~" if tilde else ""
-        return ":meth:`{}{}.{}`".format(
-            prefix, meth.__module__, meth.__qualname__
-        )
 
-    @staticmethod
-    def sphinx_ref(txt: str, label: str = None, suffix: str = "") -> str:
-        return f":ref:`{txt} <{label}{suffix}>`"
+# ==== HELPER FUNCTIONS ==== #
+
+
+def sphinx_class(*, klass: Type[Any], tilde: bool = True) -> str:
+    prefix = "~" if tilde else ""
+    return f":class:`{prefix}{klass.__module__}.{klass.__name__}`"
+
+
+def sphinx_method(*, method: Callable[..., Any], tilde: bool = True) -> str:
+    prefix = "~" if tilde else ""
+    return ":meth:`{}{}.{}`".format(
+        prefix, method.__module__, method.__qualname__
+    )
+
+
+def sphinx_ref(
+    *, txt: str, label: Optional[str] = None, suffix: str = ""
+) -> str:
+    return f":ref:`{txt} <{label}{suffix}>`"
+
 
-    @staticmethod
-    def sphinx_link(txt):
-        return "`{}`_".format(txt)
+def sphinx_link(*, txt: str) -> str:
+    return f"`{txt}`_"
diff --git a/doc/source/scripts/core.py b/doc/source/scripts/core.py
@@ -1,8 +1,3 @@
-from __future__ import print_function
-
-import os
-
-import tabulate
 from MDAnalysis import _TOPOLOGY_ATTRS
 
 # ====== TOPOLOGY ====== #
@@ -75,11 +70,9 @@
     for c in _TOPOLOGY_ATTRS.values()
 }
 
-base_attrnames = set(
-    ["atomattrs", "residueattrs", "segmentattrs", "topologyattrs"]
-)
+base_attrnames = {"atomattrs", "residueattrs", "segmentattrs", "topologyattrs"}
 
-core_attrnames = set(["indices", "resindices", "segindices"])
+core_attrnames = {"indices", "resindices", "segindices"}
 
 BASE_ATTRS = {k: v for k, v in ATTRS.items() if k in base_attrnames}
 
@@ -90,12 +83,6 @@
 }
 
 TOPOLOGY_CLS = sorted(
-    set(
-        [
-            x
-            for x in _TOPOLOGY_ATTRS.values()
-            if x.attrname in NON_CORE_ATTRS.keys()
-        ]
-    ),
+    {x for x in _TOPOLOGY_ATTRS.values() if x.attrname in NON_CORE_ATTRS},
     key=lambda x: x.attrname,
 )