Post processors priority (#726)

* Adopt priority based sorting for post-processors

* Setup the default post processor inside the PriorityProcessor with priority 200.

README.rst

@@ -76,6 +76,9 @@ What's New?
 in development
 ^^^^^^^^^^^^^^
 
+* `ExtRegistrar.register_post_processor()` now supports a `priority` argument that is an int.
+  Highest priority callables will be called first during post-processing.
+
 pydoctor 23.9.0
 ^^^^^^^^^^^^^^^
 

pydoctor/astbuilder.py

@@ -1292,3 +1292,4 @@ def parseDocformat(node: ast.Assign, mod: model.Module) -> None:
 
 def setup_pydoctor_extension(r:extensions.ExtRegistrar) -> None:
     r.register_astbuilder_visitor(TypeAliasVisitorExt)
+    r.register_post_processor(model.defaultPostProcess, priority=200)

pydoctor/extensions/__init__.py

@@ -5,7 +5,7 @@
 """
 import importlib
 import sys
-from typing import Any, Callable, Dict, Iterable, Iterator, List, Type, Union, TYPE_CHECKING, cast
+from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Type, Union, TYPE_CHECKING, cast
 
 # In newer Python versions, use importlib.resources from the standard library.
 # On older versions, a compatibility package must be installed from PyPI.
@@ -111,6 +111,59 @@ def _get_mixins(*mixins: Type[MixinT]) -> Dict[str, List[Type[MixinT]]]:
             assert False, f"Invalid mixin {mixin.__name__!r}. Mixins must subclass one of the base class."
     return mixins_by_name
 
+# Largely inspired by docutils Transformer class.
+DEFAULT_PRIORITY = 100
+class PriorityProcessor:
+    """
+    Stores L{Callable} and applies them to the system based on priority or insertion order.
+    The default priority is C{100}, see code source of L{astbuilder.setup_pydoctor_extension}, 
+    and others C{setup_pydoctor_extension} functions.
+
+    Highest priority callables will be called first, when priority is the same it's FIFO order.
+
+    One L{PriorityProcessor} should only be run once on the system.
+    """
+
+    def __init__(self, system:'model.System'):
+        self.system = system
+        self.applied: List[Callable[['model.System'], None]] = []
+        self._post_processors: List[Tuple[object, Callable[['model.System'], None]]] = []
+        self._counter = 256
+        """Internal counter to keep track of the add order of callables."""
+
+    def add_post_processor(self, post_processor:Callable[['model.System'], None], 
+                           priority:Optional[int]) -> None:
+        if priority is None:
+            priority = DEFAULT_PRIORITY
+        priority_key = self._get_priority_key(priority)
+        self._post_processors.append((priority_key, post_processor))
+
+    def _get_priority_key(self, priority:int) -> object:
+        """
+        Return a tuple, `priority` combined with `self._counter`.
+
+        This ensures FIFO order on callables with identical priority.
+        """
+        self._counter -= 1
+        return (priority, self._counter)
+
+    def apply_processors(self) -> None:
+        """Apply all of the stored processors, in priority order."""
+        if self.applied:
+            # this is typically only reached in tests, when we 
+            # call fromText() several times with the same 
+            # system or when we manually call System.postProcess()
+            self.system.msg('post processing', 
+                            'warning: multiple post-processing pass detected', 
+                            thresh=-1)
+            self.applied.clear()
+
+        self._post_processors.sort()
+        for p in reversed(self._post_processors):
+            _, post_processor = p
+            post_processor(self.system)
+            self.applied.append(post_processor)
+
 @attr.s(auto_attribs=True)
 class ExtRegistrar:
     """
@@ -133,14 +186,18 @@ def register_astbuilder_visitor(self,
         self.system._astbuilder_visitors.extend(visitor)
 
     def register_post_processor(self, 
-            *post_processor: Callable[['model.System'], None]) -> None:
+            *post_processor: Callable[['model.System'], None], 
+            priority:Optional[int]=None) -> None:
         """
         Register post processor(s).
          
         A post-processor is simply a one-argument callable receiving 
         the processed L{model.System} and doing stuff on the L{model.Documentable} tree.
+
+        @param priority: See L{PriorityProcessor}.
         """
-        self.system._post_processors.extend(post_processor)
+        for p in post_processor:
+            self.system._post_processor.add_post_processor(p, priority)
 
 def load_extension_module(system:'model.System', mod: str) -> None:
     """

pydoctor/model.py

@@ -20,7 +20,7 @@
 from inspect import signature, Signature
 from pathlib import Path
 from typing import (
-    TYPE_CHECKING, Any, Callable, Collection, Dict, Iterator, List, Mapping,
+    TYPE_CHECKING, Any, Collection, Dict, Iterator, List, Mapping,
     Optional, Sequence, Set, Tuple, Type, TypeVar, Union, cast, overload
 )
 from urllib.parse import quote
@@ -962,7 +962,7 @@ def __init__(self, options: Optional['Options'] = None):
         # Initialize the extension system
         self._factory = factory.Factory()
         self._astbuilder_visitors: List[Type['astutils.NodeVisitorExt']] = []
-        self._post_processors: List[Callable[['System'], None]] = []
+        self._post_processor = extensions.PriorityProcessor(self)
 
         if self.extensions == _default_extensions:
             self.extensions = list(extensions.get_extensions())
@@ -1435,29 +1435,10 @@ def postProcess(self) -> None:
         Analysis of relations between documentables can be done here,
         without the risk of drawing incorrect conclusions because modules
         were not fully processed yet.
-        """
-        for cls in self.objectsOfType(Class):
-
-            # Initiate the MROs
-            cls._init_mro()
-            # Lookup of constructors
-            cls._init_constructors()
-
-            # Compute subclasses
-            for b in cls.baseobjects:
-                if b is not None:
-                    b.subclasses.append(cls)
-
-            # Checking whether the class is an exception
-            if is_exception(cls):
-                cls.kind = DocumentableKind.EXCEPTION
-
-        for attrib in self.objectsOfType(Attribute):
-            _inherits_instance_variable_kind(attrib)
-
-        for post_processor in self._post_processors:
-            post_processor(self)
 
+        @See: L{extensions.PriorityProcessor}.
+        """
+        self._post_processor.apply_processors()
 
     def fetchIntersphinxInventories(self, cache: CacheT) -> None:
         """
@@ -1466,6 +1447,25 @@ def fetchIntersphinxInventories(self, cache: CacheT) -> None:
         for url in self.options.intersphinx:
             self.intersphinx.update(cache, url)
 
+def defaultPostProcess(system:'System') -> None:
+    for cls in system.objectsOfType(Class):
+        # Initiate the MROs
+        cls._init_mro()
+        # Lookup of constructors
+        cls._init_constructors()
+
+        # Compute subclasses
+        for b in cls.baseobjects:
+            if b is not None:
+                b.subclasses.append(cls)
+
+        # Checking whether the class is an exception
+        if is_exception(cls):
+            cls.kind = DocumentableKind.EXCEPTION
+
+    for attrib in system.objectsOfType(Attribute):
+       _inherits_instance_variable_kind(attrib)
+
 def _inherits_instance_variable_kind(attr: Attribute) -> None:
     """
     If any of the inherited members of a class variable is an instance variable,

pydoctor/test/test_astbuilder.py

@@ -381,9 +381,9 @@ def test_relative_import_past_top(
     ''', modname='mod', parent_name='pkg', system=system)
     captured = capsys.readouterr().out
     if level == 1:
-        assert not captured
+        assert 'relative import level' not in captured
     else:
-        assert f'pkg.mod:2: relative import level ({level}) too high\n' == captured
+        assert f'pkg.mod:2: relative import level ({level}) too high\n' in captured
 
 @systemcls_param
 def test_class_with_base_from_module(systemcls: Type[model.System]) -> None:

pydoctor/test/test_epydoc2stan.py

@@ -1482,6 +1482,9 @@ def test_module_docformat(capsys: CapSys) -> None:
     captured = capsys.readouterr().out
     assert not captured
 
+    system = model.System()
+    system.options.docformat = 'epytext'
+
     mod = fromText('''
     """
     Link to pydoctor: `pydoctor <https://github.com/twisted/pydoctor>`_.
@@ -1520,14 +1523,18 @@ def f(a: str, b: int):
 
     system = model.System()
     system.options.docformat = 'restructuredtext'
-    top = fromText(top_src, modname='top', is_package=True, system=system)
-    fromText(pkg_src, modname='pkg', parent_name='top', is_package=True,
-                   system=system)
-    mod = fromText(mod_src, modname='top.pkg.mod', parent_name='top.pkg', system=system)
+    builder = system.systemBuilder(system)
+    builder.addModuleString(top_src, modname='top', is_package=True)
+    builder.addModuleString(pkg_src, modname='pkg', parent_name='top', is_package=True)
+    builder.addModuleString(mod_src, modname='mod', parent_name='top.pkg')
+    builder.buildModules()
 
+    top = system.allobjects['top']
+    mod = system.allobjects['top.pkg.mod']
+    assert isinstance(mod, model.Module)
+    assert mod.docformat == 'epytext'
     captured = capsys.readouterr().out
     assert not captured
-
     assert ''.join(docstring2html(top.contents['f']).splitlines()) == ''.join(docstring2html(mod.contents['f']).splitlines())
 
 
@@ -1553,10 +1560,14 @@ def f(self, a: str, b: int):
     '''
 
     system = model.System()
+    builder = system.systemBuilder(system)
     system.options.docformat = 'epytext'
 
-    mod = fromText(mod_src, modname='mod', system=system)
-    mod2 = fromText(mod2_src, modname='mod2', system=system)
+    builder.addModuleString(mod_src, modname='mod',)
+    builder.addModuleString(mod2_src, modname='mod2',)
+    builder.buildModules()
+    mod = system.allobjects['mod']
+    mod2 = system.allobjects['mod2']
 
     captured = capsys.readouterr().out
     assert not captured
@@ -1610,11 +1621,14 @@ def f(a, b):
     '''
 
     system = model.System()
+    builder = system.systemBuilder(system)
     system.options.docformat = 'restructuredtext'
 
-    fromText("", modname='pack', system=system, is_package=True)
-    fromText(mod1, modname='mod1', system=system, parent_name='pack')
-    mod = fromText(mod2, modname='mod2', system=system, parent_name='pack')
+    builder.addModuleString("", modname='pack', is_package=True)
+    builder.addModuleString(mod1, modname='mod1',parent_name='pack')    
+    builder.addModuleString(mod2, modname='mod2', parent_name='pack')
+    builder.buildModules()
+    mod = system.allobjects['pack.mod2']
 
     captured = capsys.readouterr().out
     assert not captured

pydoctor/test/test_model.py

@@ -13,7 +13,7 @@
 from twisted.web.template import Tag
 
 from pydoctor.options import Options
-from pydoctor import model, stanutils
+from pydoctor import model, stanutils, extensions
 from pydoctor.templatewriter import pages
 from pydoctor.utils import parse_privacy_tuple
 from pydoctor.sphinx import CacheT
@@ -548,3 +548,28 @@ def f():...
     assert not innerFn.isNameDefined('F')
     assert not innerFn.isNameDefined('var')
     assert innerFn.isNameDefined('f')
+
+def test_priority_processor(capsys:CapSys) -> None:
+    system = model.System()
+    r = extensions.ExtRegistrar(system)
+    processor = system._post_processor
+    processor._post_processors.clear()
+
+    r.register_post_processor(lambda s:print('priority 200'), priority=200)
+    r.register_post_processor(lambda s:print('priority 100'))
+    r.register_post_processor(lambda s:print('priority 25'), priority=25)
+    r.register_post_processor(lambda s:print('priority 150'), priority=150)
+    r.register_post_processor(lambda s:print('priority 100 (bis)'))
+    r.register_post_processor(lambda s:print('priority 200 (bis)'), priority=200)
+
+    assert len(processor._post_processors)==6
+    processor.apply_processors()
+    assert len(processor.applied)==6
+
+    assert capsys.readouterr().out.strip().splitlines() == ['priority 200', 
+                                                            'priority 200 (bis)',
+                                                            'priority 150',
+                                                            'priority 100',
+                                                            'priority 100 (bis)',
+                                                            'priority 25',
+                                                            ]