From 7f820289c792ae930689ee81e4be546713f535d9 Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Tue, 26 Mar 2024 17:32:09 -0400 Subject: [PATCH] Allow to choose members order of classes and modules (#653) * Fix #580 * Add options --cls-member-order and --mod-member-order, fix #485 * Introduce System.membersOrder to be able to further customize the presentation order of members. * Give priority to the line numbers coming from AST analysis over the ones from docstring fields. * Keep track of the line number of the docstring to report link not found warnings at the correct location. --- README.rst | 4 + pydoctor/epydoc2stan.py | 11 ++- pydoctor/model.py | 43 +++++++++- pydoctor/options.py | 10 +++ pydoctor/templatewriter/pages/__init__.py | 42 +++------- pydoctor/templatewriter/pages/sidebar.py | 5 +- pydoctor/templatewriter/summary.py | 8 +- pydoctor/templatewriter/util.py | 49 ++++++++++-- pydoctor/test/test_templatewriter.py | 98 ++++++++++++++++++++++- 9 files changed, 218 insertions(+), 52 deletions(-) diff --git a/README.rst b/README.rst index d8aec0972..5220f27bd 100644 --- a/README.rst +++ b/README.rst @@ -84,6 +84,10 @@ This is the last major release to support Python 3.7. Highest priority callables will be called first during post-processing. * Fix too noisy ``--verbose`` mode (suppres some ambiguous annotations warnings). * Fix type processing inside restructuredtext consolidated fields. +* Add options ``--cls-member-order`` and ``--mod-member-order`` to customize the presentation + order of class members and module/package members, the supported values are "alphabetical" or "source". + The default behavior is to sort all members alphabetically. +* Make sure the line number coming from ast analysis has precedence over the line of a ``ivar`` field. pydoctor 23.9.1 ^^^^^^^^^^^^^^^ diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index 1b2e1b069..395c8bf25 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -934,7 +934,10 @@ def extract_fields(obj: model.CanContainImportsDocumentable) -> None: attrobj.kind = None attrobj.parentMod = obj.parentMod obj.system.addObject(attrobj) - attrobj.setLineNumber(obj.docstring_lineno + field.lineno) + lineno = model.LineFromDocstringField(obj.docstring_lineno + field.lineno) + attrobj.setLineNumber(lineno) + if not attrobj.docstring_lineno: + attrobj.docstring_lineno = lineno if tag == 'type': attrobj.parsed_type = field.body() else: @@ -1137,10 +1140,12 @@ def populate_constructors_extra_info(cls:model.Class) -> None: if constructors: plural = 's' if len(constructors)>1 else '' extra_epytext = f'Constructor{plural}: ' - for i, c in enumerate(sorted(constructors, key=util.objects_order)): + for i, c in enumerate(sorted(constructors, + key=util.alphabetical_order_func)): if i != 0: extra_epytext += ', ' short_text = format_constructor_short_text(c, cls) extra_epytext += '`%s <%s>`' % (short_text, c.fullName()) - cls.extra_info.append(parse_docstring(cls, extra_epytext, cls, 'restructuredtext', section='constructor extra')) + cls.extra_info.append(parse_docstring( + cls, extra_epytext, cls, 'restructuredtext', section='constructor extra')) diff --git a/pydoctor/model.py b/pydoctor/model.py index 61f84a2bf..31c30ac91 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -21,7 +21,7 @@ from inspect import signature, Signature from pathlib import Path from typing import ( - TYPE_CHECKING, Any, Collection, Dict, Iterator, List, Mapping, + TYPE_CHECKING, Any, Collection, Dict, Iterator, List, Mapping, Callable, Optional, Sequence, Set, Tuple, Type, TypeVar, Union, cast, overload ) from urllib.parse import quote @@ -59,6 +59,11 @@ line in the string, rather than the first line. """ +class LineFromAst(int): + "Simple L{int} wrapper for linenumbers coming from ast analysis." + +class LineFromDocstringField(int): + "Simple L{int} wrapper for linenumbers coming from docstrings." class DocLocation(Enum): OWN_PAGE = 1 @@ -126,7 +131,7 @@ class Documentable: parsed_summary: Optional[ParsedDocstring] = None parsed_type: Optional[ParsedDocstring] = None docstring_lineno = 0 - linenumber = 0 + linenumber: LineFromAst | LineFromDocstringField | Literal[0] = 0 sourceHref: Optional[str] = None kind: Optional[DocumentableKind] = None @@ -164,8 +169,24 @@ def setDocstring(self, node: astutils.Str) -> None: self.docstring = doc self.docstring_lineno = lineno - def setLineNumber(self, lineno: int) -> None: - if not self.linenumber: + def setLineNumber(self, lineno: LineFromDocstringField | LineFromAst | int) -> None: + """ + Save the linenumber of this object. + + If the linenumber is already set from a ast analysis, this is an no-op. + If the linenumber is already set from docstring fields and the new linenumber + if not from docstring fields as well, the old docstring based linumber will be replaced + with the one from ast analysis since this takes precedence. + + @param lineno: The linenumber. + If the given linenumber is simply an L{int} we'll assume it's coming from the ast builder + and it will be converted to an L{LineFromAst} instance. + """ + if not self.linenumber or ( + isinstance(self.linenumber, LineFromDocstringField) + and not isinstance(lineno, LineFromDocstringField)): + if not isinstance(lineno, (LineFromAst, LineFromDocstringField)): + lineno = LineFromAst(lineno) self.linenumber = lineno parentMod = self.parentMod if parentMod is not None: @@ -1133,6 +1154,20 @@ def privacyClass(self, ob: Documentable) -> PrivacyClass: self._privacyClassCache[ob_fullName] = privacy return privacy + def membersOrder(self, ob: Documentable) -> Callable[[Documentable], Tuple[Any, ...]]: + """ + Returns a callable suitable to be used with L{sorted} function. + Used to sort the given object's members for presentation. + + Users can customize class and module members order independently, or can override this method + with a custom system class for further tweaks. + """ + from pydoctor.templatewriter.util import objects_order + if isinstance(ob, Class): + return objects_order(self.options.cls_member_order) + else: + return objects_order(self.options.mod_member_order) + def addObject(self, obj: Documentable) -> None: """Add C{object} to the system.""" diff --git a/pydoctor/options.py b/pydoctor/options.py index 272539dce..60cebc1ae 100644 --- a/pydoctor/options.py +++ b/pydoctor/options.py @@ -21,6 +21,7 @@ from pydoctor._configparser import CompositeConfigParser, IniConfigParser, TomlConfigParser, ValidatorParser if TYPE_CHECKING: + from typing import Literal from pydoctor import model from pydoctor.templatewriter import IWriter @@ -236,6 +237,13 @@ def get_parser() -> ArgumentParser: parser.add_argument( '--system-class', dest='systemclass', default=DEFAULT_SYSTEM, help=("A dotted name of the class to use to make a system.")) + + parser.add_argument( + '--cls-member-order', dest='cls_member_order', default="alphabetical", choices=["alphabetical", "source"], + help=("Presentation order of class members. (default: alphabetical)")) + parser.add_argument( + '--mod-member-order', dest='mod_member_order', default="alphabetical", choices=["alphabetical", "source"], + help=("Presentation order of module/package members. (default: alphabetical)")) parser.add_argument('-V', '--version', action='version', version=f'%(prog)s {__version__}') @@ -365,6 +373,8 @@ class Options: sidebarexpanddepth: int = attr.ib() sidebartocdepth: int = attr.ib() nosidebar: int = attr.ib() + cls_member_order: 'Literal["alphabetical", "source"]' = attr.ib() + mod_member_order: 'Literal["alphabetical", "source"]' = attr.ib() def __attrs_post_init__(self) -> None: # do some validations... diff --git a/pydoctor/templatewriter/pages/__init__.py b/pydoctor/templatewriter/pages/__init__.py index efc8bb80f..2f57084c0 100644 --- a/pydoctor/templatewriter/pages/__init__.py +++ b/pydoctor/templatewriter/pages/__init__.py @@ -3,7 +3,7 @@ from typing import ( TYPE_CHECKING, Dict, Iterator, List, Optional, Mapping, Sequence, - Tuple, Type, Union + Type, Union ) import ast import abc @@ -27,25 +27,6 @@ from pydoctor.templatewriter.pages.functionchild import FunctionChild -def objects_order(o: model.Documentable) -> Tuple[int, int, str]: - """ - Function to use as the value of standard library's L{sorted} function C{key} argument - such that the objects are sorted by: Privacy, Kind and Name. - - Example:: - - children = sorted((o for o in ob.contents.values() if o.isVisible), - key=objects_order) - """ - - def map_kind(kind: model.DocumentableKind) -> model.DocumentableKind: - if kind == model.DocumentableKind.PACKAGE: - # packages and modules should be listed together - return model.DocumentableKind.MODULE - return kind - - return (-o.privacyClass.value, -map_kind(o.kind).value if o.kind else 0, o.fullName().lower()) - def format_decorators(obj: Union[model.Function, model.Attribute, model.FunctionOverload]) -> Iterator["Flattenable"]: # Since we use this function to colorize the FunctionOverload decorators and it's not an actual Documentable subclass, we use the overload's # primary function for parts that requires an interface to Documentable methods or attributes @@ -246,6 +227,7 @@ def __init__(self, ob: model.Documentable, template_lookup: TemplateLookup, docg if docgetter is None: docgetter = util.DocGetter() self.docgetter = docgetter + self._order = ob.system.membersOrder(ob) @property def page_url(self) -> str: @@ -301,7 +283,7 @@ def docstring(self) -> "Flattenable": def children(self) -> Sequence[model.Documentable]: return sorted( (o for o in self.ob.contents.values() if o.isVisible), - key=util.objects_order) + key=self._order) def packageInitTable(self) -> "Flattenable": return () @@ -321,7 +303,7 @@ def mainTable(self) -> "Flattenable": def methods(self) -> Sequence[model.Documentable]: return sorted((o for o in self.ob.contents.values() if o.documentation_location is model.DocLocation.PARENT_PAGE and o.isVisible), - key=util.objects_order) + key=self._order) def childlist(self) -> List[Union["AttributeChild", "FunctionChild"]]: from pydoctor.templatewriter.pages.attributechild import AttributeChild @@ -398,13 +380,13 @@ def extras(self) -> List["Flattenable"]: class PackagePage(ModulePage): def children(self) -> Sequence[model.Documentable]: - return sorted(self.ob.submodules(), key=objects_order) + return sorted(self.ob.submodules(), key=self._order) def packageInitTable(self) -> "Flattenable": children = sorted( (o for o in self.ob.contents.values() if not isinstance(o, model.Module) and o.isVisible), - key=util.objects_order) + key=self._order) if children: loader = ChildTable.lookup_loader(self.template_lookup) return [ @@ -415,9 +397,9 @@ def packageInitTable(self) -> "Flattenable": return () def methods(self) -> Sequence[model.Documentable]: - return [o for o in self.ob.contents.values() + return sorted([o for o in self.ob.contents.values() if o.documentation_location is model.DocLocation.PARENT_PAGE - and o.isVisible] + and o.isVisible], key=self._order) def assembleList( system: model.System, @@ -480,7 +462,7 @@ def extras(self) -> List["Flattenable"]: self.classSignature(), ":", source ), class_='class-signature')) - subclasses = sorted(self.ob.subclasses, key=util.objects_order) + subclasses = sorted(self.ob.subclasses, key=util.alphabetical_order_func) if subclasses: p = assembleList(self.ob.system, "Known subclasses: ", [o.fullName() for o in subclasses], self.page_url) @@ -508,7 +490,7 @@ def baseTables(self, request: object, item: Tag) -> "Flattenable": return [item.clone().fillSlots( baseName=self.baseName(b), baseTable=ChildTable(self.docgetter, self.ob, - sorted(attrs, key=util.objects_order), + sorted(attrs, key=self._order), loader)) for b, attrs in baselists] @@ -542,7 +524,7 @@ def get_override_info(cls:model.Class, member_name:str, page_url:Optional[str]=N 'overrides ', tags.code(epydoc2stan.taglink(overridden, page_url))) break - ocs = sorted(util.overriding_subclasses(cls, member_name), key=util.objects_order) + ocs = sorted(util.overriding_subclasses(cls, member_name), key=util.alphabetical_order_func) if ocs: l = assembleList(cls.system, 'overridden in ', [o.fullName() for o in ocs], page_url) @@ -557,7 +539,7 @@ def extras(self) -> List["Flattenable"]: r = super().extras() if self.ob.isinterface: namelist = [o.fullName() for o in - sorted(self.ob.implementedby_directly, key=util.objects_order)] + sorted(self.ob.implementedby_directly, key=util.alphabetical_order_func)] label = 'Known implementations: ' else: namelist = sorted(self.ob.implements_directly, key=lambda x:x.lower()) diff --git a/pydoctor/templatewriter/pages/sidebar.py b/pydoctor/templatewriter/pages/sidebar.py index 9d3c181d2..61ba10979 100644 --- a/pydoctor/templatewriter/pages/sidebar.py +++ b/pydoctor/templatewriter/pages/sidebar.py @@ -126,6 +126,7 @@ def __init__(self, loader: ITemplateLoader, ob: Documentable, documented_ob: Doc self.documented_ob = documented_ob self.template_lookup = template_lookup + self._order = ob.system.membersOrder(ob) self._depth = depth self._level = level + 1 @@ -172,10 +173,10 @@ def _children(self, inherited: bool = False) -> List[Documentable]: if inherited: assert isinstance(self.ob, Class), "Use inherited=True only with Class instances" return sorted((o for o in util.inherited_members(self.ob) if o.isVisible), - key=util.objects_order) + key=self._order) else: return sorted((o for o in self.ob.contents.values() if o.isVisible), - key=util.objects_order) + key=self._order) def _isExpandable(self, list_type: Type[Documentable]) -> bool: """ diff --git a/pydoctor/templatewriter/summary.py b/pydoctor/templatewriter/summary.py index 73bebe401..36bd5adea 100644 --- a/pydoctor/templatewriter/summary.py +++ b/pydoctor/templatewriter/summary.py @@ -10,8 +10,8 @@ from twisted.web.template import Element, Tag, TagLoader, renderer, tags from pydoctor import epydoc2stan, model, linker -from pydoctor.templatewriter import TemplateLookup -from pydoctor.templatewriter.pages import Page, objects_order +from pydoctor.templatewriter import TemplateLookup, util +from pydoctor.templatewriter.pages import Page if TYPE_CHECKING: from twisted.web.template import Flattenable @@ -36,7 +36,7 @@ def moduleSummary(module: model.Module, page_url: str) -> Tag: # If there are more than 50 modules and no submodule has # further submodules we use a more compact presentation. li = tags.li(class_='compact-modules') - for m in sorted(contents, key=objects_order): + for m in sorted(contents, key=util.alphabetical_order_func): span = tags.span() span(tags.code(linker.taglink(m, m.url, label=m.name))) span(', ') @@ -47,7 +47,7 @@ def moduleSummary(module: model.Module, page_url: str) -> Tag: li.children[-1].children.pop() # type: ignore ul(li) else: - for m in sorted(contents, key=objects_order): + for m in sorted(contents, key=util.alphabetical_order_func): ul(moduleSummary(m, page_url)) r(ul) return r diff --git a/pydoctor/templatewriter/util.py b/pydoctor/templatewriter/util.py index 2ab28ee78..902cda9f1 100644 --- a/pydoctor/templatewriter/util.py +++ b/pydoctor/templatewriter/util.py @@ -2,12 +2,15 @@ from __future__ import annotations import warnings -from typing import (Any, Dict, Generic, Iterable, Iterator, List, Mapping, - Optional, MutableMapping, Tuple, TypeVar, Union, Sequence) +from typing import (Any, Callable, Dict, Generic, Iterable, Iterator, List, Mapping, + Optional, MutableMapping, Tuple, TypeVar, Union, Sequence, TYPE_CHECKING) from pydoctor import epydoc2stan import collections.abc from pydoctor import model +if TYPE_CHECKING: + from typing import Literal + from twisted.web.template import Tag class DocGetter: @@ -84,17 +87,49 @@ def unmasked_attrs(baselist: Sequence[model.Class]) -> Sequence[model.Documentab return [o for o in baselist[0].contents.values() if o.isVisible and o.name not in maybe_masking] -def objects_order(o: model.Documentable) -> Tuple[int, int, str]: +def alphabetical_order_func(o: model.Documentable) -> Tuple[Any, ...]: + """ + Sort by privacy, kind and fullname. + Callable to use as the value of standard library's L{sorted} function C{key} argument. + """ + return (-o.privacyClass.value, -_map_kind(o.kind).value if o.kind else 0, o.fullName().lower()) + +def source_order_func(o: model.Documentable) -> Tuple[Any, ...]: + """ + Sort by privacy, kind and linenumber. + Callable to use as the value of standard library's L{sorted} function C{key} argument. """ - Function to use as the value of standard library's L{sorted} function C{key} argument - such that the objects are sorted by: Privacy, Kind and Name. + if isinstance(o, model.Module): + # Still sort modules by name since they all have the same linenumber. + return (-o.privacyClass.value, -_map_kind(o.kind).value if o.kind else 0, o.fullName().lower()) + else: + return (-o.privacyClass.value, -_map_kind(o.kind).value if o.kind else 0, o.linenumber) + # last implicit orderring is the order of insertion. + +def _map_kind(kind: model.DocumentableKind) -> model.DocumentableKind: + if kind == model.DocumentableKind.PACKAGE: + # packages and modules should be listed together + return model.DocumentableKind.MODULE + return kind + +def objects_order(order: 'Literal["alphabetical", "source"]') -> Callable[[model.Documentable], Tuple[Any, ...]]: + """ + Function to craft a callable to use as the value of standard library's L{sorted} function C{key} argument + such that the objects are sorted by: Privacy, Kind first, then by Name or Linenumber depending on + C{order} argument. Example:: children = sorted((o for o in ob.contents.values() if o.isVisible), - key=objects_order) + key=objects_order("alphabetical")) """ - return (-o.privacyClass.value, -o.kind.value if o.kind else 0, o.fullName().lower()) + + if order == "alphabetical": + return alphabetical_order_func + elif order == "source": + return source_order_func + else: + assert False def class_members(cls: model.Class) -> List[Tuple[Tuple[model.Class, ...], Sequence[model.Documentable]]]: """ diff --git a/pydoctor/test/test_templatewriter.py b/pydoctor/test/test_templatewriter.py index 64a331474..dbc143967 100644 --- a/pydoctor/test/test_templatewriter.py +++ b/pydoctor/test/test_templatewriter.py @@ -644,7 +644,8 @@ def test_index_contains_infos(tmp_path: Path) -> None: for i in infos: assert i in page, page -def test_objects_order_mixed_modules_and_packages() -> None: +@pytest.mark.parametrize('_order', ["alphabetical", "source"]) +def test_objects_order_mixed_modules_and_packages(_order:str) -> None: """ Packages and modules are mixed when sorting with objects_order. """ @@ -655,11 +656,104 @@ def test_objects_order_mixed_modules_and_packages() -> None: fromText('', parent_name='top', modname='bbb', system=system) fromText('', parent_name='top', modname='aba', system=system, is_package=True) - _sorted = sorted(top.contents.values(), key=pages.objects_order) + _sorted = sorted(top.contents.values(), key=util.objects_order(_order)) # type:ignore names = [s.name for s in _sorted] assert names == ['aaa', 'aba', 'bbb'] +def test_change_member_order() -> None: + """ + Default behaviour is to sort everything by privacy, kind and then by name. + But we allow to customize the class and modules members independendly, + the reason for this is to permit to match rustdoc behaviour, + that is to sort class members by source, the rest by name. + """ + system = model.System() + assert system.options.cls_member_order == system.options.mod_member_order == "alphabetical" + + mod = fromText('''\ + class Foo: + def start():... + def process_link():... + def process_emphasis():... + def process_blockquote():... + def process_table():... + def end():... + + class Bar:... + + b,a = 1,2 + ''', system=system) + + _sorted = sorted(mod.contents.values(), key=system.membersOrder(mod)) + assert [s.name for s in _sorted] == ['Bar', 'Foo', 'a', 'b'] # default ordering is alphabetical + + system.options.mod_member_order = 'source' + _sorted = sorted(mod.contents.values(), key=system.membersOrder(mod)) + assert [s.name for s in _sorted] == ['Foo', 'Bar', 'b', 'a'] + + Foo = mod.contents['Foo'] + + _sorted = sorted(Foo.contents.values(), key=system.membersOrder(Foo)) + names = [s.name for s in _sorted] + + assert names ==['end', + 'process_blockquote', + 'process_emphasis', + 'process_link', + 'process_table', + 'start',] + + system.options.cls_member_order = "source" + _sorted = sorted(Foo.contents.values(), key=system.membersOrder(Foo)) + names = [s.name for s in _sorted] + + assert names == ['start', + 'process_link', + 'process_emphasis', + 'process_blockquote', + 'process_table', + 'end'] + +def test_ivar_field_order_precedence(capsys: CapSys) -> None: + """ + We special case the linen umber coming from docstring fields such that they can get overriden + by AST linenumber. + """ + system = model.System(model.Options.from_args(['--cls-member-order=source'])) + mod = fromText(''' + import attr + __docformat__ = 'restructuredtext' + @attr.s + class Foo: + """ + :ivar a: `broken1 <>`_ Thing. + :ivar b: `broken2 <>`_ Stuff. + """ + + b = attr.ib() + a = attr.ib() + ''', system=system) + + Foo = mod.contents['Foo'] + getHTMLOf(Foo) + assert Foo.docstring_lineno == 7 + + assert Foo.parsed_docstring.fields[0].lineno == 0 # type:ignore + assert Foo.parsed_docstring.fields[1].lineno == 1 # type:ignore + + assert Foo.contents['a'].linenumber == 12 + assert Foo.contents['b'].linenumber == 11 + + assert Foo.contents['a'].docstring_lineno == 7 + assert Foo.contents['b'].docstring_lineno == 8 + + _sorted = sorted(Foo.contents.values(), key=system.membersOrder(Foo)) + names = [s.name for s in _sorted] + + assert names == ['b', 'a'] # should be 'b', 'a'. + + src_crash_xml_entities = '''\ """ These are non-breaking spaces