Allow to choose members order of classes and modules (#653)

* Fix #580

* Add options --cls-member-order and --mod-member-order, fix #485

* Introduce System.membersOrder to be able to further customize the presentation order of members.

* Give priority to the line numbers coming from AST analysis over the ones from docstring fields.

* Keep track of the line number of the docstring to report link not found warnings at the correct location.

README.rst

@@ -84,6 +84,10 @@ This is the last major release to support Python 3.7.
   Highest priority callables will be called first during post-processing.
 * Fix too noisy ``--verbose`` mode (suppres some ambiguous annotations warnings).
 * Fix type processing inside restructuredtext consolidated fields.
+* Add options ``--cls-member-order`` and ``--mod-member-order`` to customize the presentation
+  order of class members and module/package members, the supported values are "alphabetical" or "source".
+  The default behavior is to sort all members alphabetically.
+* Make sure the line number coming from ast analysis has precedence over the line of a ``ivar`` field.
 
 pydoctor 23.9.1
 ^^^^^^^^^^^^^^^

pydoctor/epydoc2stan.py

@@ -934,7 +934,10 @@ def extract_fields(obj: model.CanContainImportsDocumentable) -> None:
                 attrobj.kind = None
                 attrobj.parentMod = obj.parentMod
                 obj.system.addObject(attrobj)
-            attrobj.setLineNumber(obj.docstring_lineno + field.lineno)
+            lineno = model.LineFromDocstringField(obj.docstring_lineno + field.lineno)
+            attrobj.setLineNumber(lineno)
+            if not attrobj.docstring_lineno:
+                attrobj.docstring_lineno = lineno
             if tag == 'type':
                 attrobj.parsed_type = field.body()
             else:
@@ -1137,10 +1140,12 @@ def populate_constructors_extra_info(cls:model.Class) -> None:
     if constructors:
         plural = 's' if len(constructors)>1 else ''
         extra_epytext = f'Constructor{plural}: '
-        for i, c in enumerate(sorted(constructors, key=util.objects_order)):
+        for i, c in enumerate(sorted(constructors, 
+                        key=util.alphabetical_order_func)):
             if i != 0:
                 extra_epytext += ', '
             short_text = format_constructor_short_text(c, cls)
             extra_epytext += '`%s <%s>`' % (short_text, c.fullName())
 
-        cls.extra_info.append(parse_docstring(cls, extra_epytext, cls, 'restructuredtext', section='constructor extra'))
+        cls.extra_info.append(parse_docstring(
+            cls, extra_epytext, cls, 'restructuredtext', section='constructor extra'))

pydoctor/model.py

@@ -21,7 +21,7 @@
 from inspect import signature, Signature
 from pathlib import Path
 from typing import (
-    TYPE_CHECKING, Any, Collection, Dict, Iterator, List, Mapping,
+    TYPE_CHECKING, Any, Collection, Dict, Iterator, List, Mapping, Callable, 
     Optional, Sequence, Set, Tuple, Type, TypeVar, Union, cast, overload
 )
 from urllib.parse import quote
@@ -59,6 +59,11 @@
 line in the string, rather than the first line.
 """
 
+class LineFromAst(int):
+    "Simple L{int} wrapper for linenumbers coming from ast analysis."
+
+class LineFromDocstringField(int):
+    "Simple L{int} wrapper for linenumbers coming from docstrings."
 
 class DocLocation(Enum):
     OWN_PAGE = 1
@@ -126,7 +131,7 @@ class Documentable:
     parsed_summary: Optional[ParsedDocstring] = None
     parsed_type: Optional[ParsedDocstring] = None
     docstring_lineno = 0
-    linenumber = 0
+    linenumber: LineFromAst | LineFromDocstringField | Literal[0] = 0
     sourceHref: Optional[str] = None
     kind: Optional[DocumentableKind] = None
 
@@ -164,8 +169,24 @@ def setDocstring(self, node: astutils.Str) -> None:
         self.docstring = doc
         self.docstring_lineno = lineno
 
-    def setLineNumber(self, lineno: int) -> None:
-        if not self.linenumber:
+    def setLineNumber(self, lineno: LineFromDocstringField | LineFromAst | int) -> None:
+        """
+        Save the linenumber of this object.
+
+        If the linenumber is already set from a ast analysis, this is an no-op.
+        If the linenumber is already set from docstring fields and the new linenumber
+        if not from docstring fields as well, the old docstring based linumber will be replaced
+        with the one from ast analysis since this takes precedence.
+
+        @param lineno: The linenumber. 
+            If the given linenumber is simply an L{int} we'll assume it's coming from the ast builder 
+            and it will be converted to an L{LineFromAst} instance.
+        """
+        if not self.linenumber or (
+            isinstance(self.linenumber, LineFromDocstringField) 
+                and not isinstance(lineno, LineFromDocstringField)):
+            if not isinstance(lineno, (LineFromAst, LineFromDocstringField)):
+                lineno = LineFromAst(lineno)
             self.linenumber = lineno
             parentMod = self.parentMod
             if parentMod is not None:
@@ -1133,6 +1154,20 @@ def privacyClass(self, ob: Documentable) -> PrivacyClass:
         self._privacyClassCache[ob_fullName] = privacy
         return privacy
 
+    def membersOrder(self, ob: Documentable) -> Callable[[Documentable], Tuple[Any, ...]]:
+        """
+        Returns a callable suitable to be used with L{sorted} function. 
+        Used to sort the given object's members for presentation.
+
+        Users can customize class and module members order independently, or can override this method
+        with a custom system class for further tweaks.
+        """
+        from pydoctor.templatewriter.util import objects_order
+        if isinstance(ob, Class):
+            return objects_order(self.options.cls_member_order)
+        else:
+            return objects_order(self.options.mod_member_order)
+
     def addObject(self, obj: Documentable) -> None:
         """Add C{object} to the system."""
 

pydoctor/options.py

@@ -21,6 +21,7 @@
 from pydoctor._configparser import CompositeConfigParser, IniConfigParser, TomlConfigParser, ValidatorParser
 
 if TYPE_CHECKING:
+    from typing import Literal
     from pydoctor import model
     from pydoctor.templatewriter import IWriter
 
@@ -236,6 +237,13 @@ def get_parser() -> ArgumentParser:
     parser.add_argument(
         '--system-class', dest='systemclass', default=DEFAULT_SYSTEM,
         help=("A dotted name of the class to use to make a system."))
+
+    parser.add_argument(
+        '--cls-member-order', dest='cls_member_order', default="alphabetical", choices=["alphabetical", "source"],
+        help=("Presentation order of class members. (default: alphabetical)"))
+    parser.add_argument(
+        '--mod-member-order', dest='mod_member_order', default="alphabetical", choices=["alphabetical", "source"],
+        help=("Presentation order of module/package members. (default: alphabetical)"))
 
     parser.add_argument('-V', '--version', action='version', version=f'%(prog)s {__version__}')
 
@@ -365,6 +373,8 @@ class Options:
     sidebarexpanddepth:     int                                     = attr.ib()
     sidebartocdepth:        int                                     = attr.ib()
     nosidebar:              int                                     = attr.ib()
+    cls_member_order:       'Literal["alphabetical", "source"]'     = attr.ib()
+    mod_member_order:       'Literal["alphabetical", "source"]'     = attr.ib()
 
     def __attrs_post_init__(self) -> None:
         # do some validations...

pydoctor/templatewriter/pages/__init__.py

@@ -3,7 +3,7 @@
 
 from typing import (
     TYPE_CHECKING, Dict, Iterator, List, Optional, Mapping, Sequence,
-    Tuple, Type, Union
+    Type, Union
 )
 import ast
 import abc
@@ -27,25 +27,6 @@
     from pydoctor.templatewriter.pages.functionchild import FunctionChild
 
 
-def objects_order(o: model.Documentable) -> Tuple[int, int, str]: 
-    """
-    Function to use as the value of standard library's L{sorted} function C{key} argument
-    such that the objects are sorted by: Privacy, Kind and Name.
-
-    Example::
-
-        children = sorted((o for o in ob.contents.values() if o.isVisible),
-                      key=objects_order)
-    """
-
-    def map_kind(kind: model.DocumentableKind) -> model.DocumentableKind:
-        if kind == model.DocumentableKind.PACKAGE:
-            # packages and modules should be listed together
-            return model.DocumentableKind.MODULE
-        return kind
-
-    return (-o.privacyClass.value, -map_kind(o.kind).value if o.kind else 0, o.fullName().lower())
-
 def format_decorators(obj: Union[model.Function, model.Attribute, model.FunctionOverload]) -> Iterator["Flattenable"]:
     # Since we use this function to colorize the FunctionOverload decorators and it's not an actual Documentable subclass, we use the overload's 
     # primary function for parts that requires an interface to Documentable methods or attributes
@@ -246,6 +227,7 @@ def __init__(self, ob: model.Documentable, template_lookup: TemplateLookup, docg
         if docgetter is None:
             docgetter = util.DocGetter()
         self.docgetter = docgetter
+        self._order = ob.system.membersOrder(ob)
 
     @property
     def page_url(self) -> str:
@@ -301,7 +283,7 @@ def docstring(self) -> "Flattenable":
     def children(self) -> Sequence[model.Documentable]:
         return sorted(
             (o for o in self.ob.contents.values() if o.isVisible),
-            key=util.objects_order)
+            key=self._order)
 
     def packageInitTable(self) -> "Flattenable":
         return ()
@@ -321,7 +303,7 @@ def mainTable(self) -> "Flattenable":
     def methods(self) -> Sequence[model.Documentable]:
         return sorted((o for o in self.ob.contents.values()
                        if o.documentation_location is model.DocLocation.PARENT_PAGE and o.isVisible), 
-                      key=util.objects_order)
+                      key=self._order)
 
     def childlist(self) -> List[Union["AttributeChild", "FunctionChild"]]:
         from pydoctor.templatewriter.pages.attributechild import AttributeChild
@@ -398,13 +380,13 @@ def extras(self) -> List["Flattenable"]:
 
 class PackagePage(ModulePage):
     def children(self) -> Sequence[model.Documentable]:
-        return sorted(self.ob.submodules(), key=objects_order)
+        return sorted(self.ob.submodules(), key=self._order)
 
     def packageInitTable(self) -> "Flattenable":
         children = sorted(
             (o for o in self.ob.contents.values()
              if not isinstance(o, model.Module) and o.isVisible),
-            key=util.objects_order)
+            key=self._order)
         if children:
             loader = ChildTable.lookup_loader(self.template_lookup)
             return [
@@ -415,9 +397,9 @@ def packageInitTable(self) -> "Flattenable":
             return ()
 
     def methods(self) -> Sequence[model.Documentable]:
-        return [o for o in self.ob.contents.values()
+        return sorted([o for o in self.ob.contents.values()
                 if o.documentation_location is model.DocLocation.PARENT_PAGE
-                and o.isVisible]
+                and o.isVisible], key=self._order)
 
 def assembleList(
         system: model.System,
@@ -480,7 +462,7 @@ def extras(self) -> List["Flattenable"]:
             self.classSignature(), ":", source
             ), class_='class-signature'))
 
-        subclasses = sorted(self.ob.subclasses, key=util.objects_order)
+        subclasses = sorted(self.ob.subclasses, key=util.alphabetical_order_func)
         if subclasses:
             p = assembleList(self.ob.system, "Known subclasses: ",
                             [o.fullName() for o in subclasses], self.page_url)
@@ -508,7 +490,7 @@ def baseTables(self, request: object, item: Tag) -> "Flattenable":
         return [item.clone().fillSlots(
                           baseName=self.baseName(b),
                           baseTable=ChildTable(self.docgetter, self.ob,
-                                               sorted(attrs, key=util.objects_order),
+                                               sorted(attrs, key=self._order),
                                                loader))
                 for b, attrs in baselists]
 
@@ -542,7 +524,7 @@ def get_override_info(cls:model.Class, member_name:str, page_url:Optional[str]=N
             'overrides ', tags.code(epydoc2stan.taglink(overridden, page_url)))
         break
 
-    ocs = sorted(util.overriding_subclasses(cls, member_name), key=util.objects_order)
+    ocs = sorted(util.overriding_subclasses(cls, member_name), key=util.alphabetical_order_func)
     if ocs:
         l = assembleList(cls.system, 'overridden in ',
                             [o.fullName() for o in ocs], page_url)
@@ -557,7 +539,7 @@ def extras(self) -> List["Flattenable"]:
         r = super().extras()
         if self.ob.isinterface:
             namelist = [o.fullName() for o in 
-                        sorted(self.ob.implementedby_directly, key=util.objects_order)]
+                        sorted(self.ob.implementedby_directly, key=util.alphabetical_order_func)]
             label = 'Known implementations: '
         else:
             namelist = sorted(self.ob.implements_directly, key=lambda x:x.lower())

pydoctor/templatewriter/pages/sidebar.py

@@ -126,6 +126,7 @@ def __init__(self, loader: ITemplateLoader, ob: Documentable, documented_ob: Doc
         self.documented_ob = documented_ob
         self.template_lookup = template_lookup
 
+        self._order = ob.system.membersOrder(ob)
         self._depth = depth
         self._level = level + 1
 
@@ -172,10 +173,10 @@ def _children(self, inherited: bool = False) -> List[Documentable]:
         if inherited:
             assert isinstance(self.ob, Class), "Use inherited=True only with Class instances"
             return sorted((o for o in util.inherited_members(self.ob) if o.isVisible),
-                              key=util.objects_order)
+                              key=self._order)
         else:
             return sorted((o for o in self.ob.contents.values() if o.isVisible),
-                              key=util.objects_order)
+                              key=self._order)
 
     def _isExpandable(self, list_type: Type[Documentable]) -> bool:
         """

pydoctor/templatewriter/summary.py

@@ -10,8 +10,8 @@
 from twisted.web.template import Element, Tag, TagLoader, renderer, tags
 
 from pydoctor import epydoc2stan, model, linker
-from pydoctor.templatewriter import TemplateLookup
-from pydoctor.templatewriter.pages import Page, objects_order
+from pydoctor.templatewriter import TemplateLookup, util
+from pydoctor.templatewriter.pages import Page
 
 if TYPE_CHECKING:
     from twisted.web.template import Flattenable
@@ -36,7 +36,7 @@ def moduleSummary(module: model.Module, page_url: str) -> Tag:
         # If there are more than 50 modules and no submodule has
         # further submodules we use a more compact presentation.
         li = tags.li(class_='compact-modules')
-        for m in sorted(contents, key=objects_order):
+        for m in sorted(contents, key=util.alphabetical_order_func):
             span = tags.span()
             span(tags.code(linker.taglink(m, m.url, label=m.name)))
             span(', ')
@@ -47,7 +47,7 @@ def moduleSummary(module: model.Module, page_url: str) -> Tag:
         li.children[-1].children.pop() # type: ignore
         ul(li)
     else:
-        for m in sorted(contents, key=objects_order):
+        for m in sorted(contents, key=util.alphabetical_order_func):
             ul(moduleSummary(m, page_url))
     r(ul)
     return r

pydoctor/templatewriter/util.py

@@ -2,12 +2,15 @@
 from __future__ import annotations
 
 import warnings
-from typing import (Any, Dict, Generic, Iterable, Iterator, List, Mapping, 
-                    Optional, MutableMapping, Tuple, TypeVar, Union, Sequence)
+from typing import (Any, Callable, Dict, Generic, Iterable, Iterator, List, Mapping, 
+                    Optional, MutableMapping, Tuple, TypeVar, Union, Sequence, TYPE_CHECKING)
 from pydoctor import epydoc2stan
 import collections.abc
 from pydoctor import model
 
+if TYPE_CHECKING:
+    from typing import Literal
+
 from twisted.web.template import Tag
 
 class DocGetter:
@@ -84,17 +87,49 @@ def unmasked_attrs(baselist: Sequence[model.Class]) -> Sequence[model.Documentab
     return [o for o in baselist[0].contents.values()
             if o.isVisible and o.name not in maybe_masking]
 
-def objects_order(o: model.Documentable) -> Tuple[int, int, str]: 
+def alphabetical_order_func(o: model.Documentable) -> Tuple[Any, ...]:
+    """
+    Sort by privacy, kind and fullname.
+    Callable to use as the value of standard library's L{sorted} function C{key} argument.
+    """
+    return (-o.privacyClass.value, -_map_kind(o.kind).value if o.kind else 0, o.fullName().lower())
+
+def source_order_func(o: model.Documentable) -> Tuple[Any, ...]:
+    """
+    Sort by privacy, kind and linenumber.
+    Callable to use as the value of standard library's L{sorted} function C{key} argument.
     """
-    Function to use as the value of standard library's L{sorted} function C{key} argument
-    such that the objects are sorted by: Privacy, Kind and Name.
+    if isinstance(o, model.Module):
+        # Still sort modules by name since they all have the same linenumber.
+        return (-o.privacyClass.value, -_map_kind(o.kind).value if o.kind else 0, o.fullName().lower()) 
+    else:
+        return (-o.privacyClass.value, -_map_kind(o.kind).value if o.kind else 0, o.linenumber) 
+        # last implicit orderring is the order of insertion.
+
+def _map_kind(kind: model.DocumentableKind) -> model.DocumentableKind:
+    if kind == model.DocumentableKind.PACKAGE:
+        # packages and modules should be listed together
+        return model.DocumentableKind.MODULE
+    return kind
+
+def objects_order(order: 'Literal["alphabetical", "source"]') -> Callable[[model.Documentable], Tuple[Any, ...]]: 
+    """
+    Function to craft a callable to use as the value of standard library's L{sorted} function C{key} argument
+    such that the objects are sorted by: Privacy, Kind first, then by Name or Linenumber depending on
+    C{order} argument.
 
     Example::
 
         children = sorted((o for o in ob.contents.values() if o.isVisible),
-                      key=objects_order)
+                      key=objects_order("alphabetical"))
     """
-    return (-o.privacyClass.value, -o.kind.value if o.kind else 0, o.fullName().lower())
+
+    if order == "alphabetical":
+        return alphabetical_order_func
+    elif order == "source":
+        return source_order_func
+    else:
+        assert False
 
 def class_members(cls: model.Class) -> List[Tuple[Tuple[model.Class, ...], Sequence[model.Documentable]]]:
     """