diff --git a/README.rst b/README.rst index 15f584be7..0601be05b 100644 --- a/README.rst +++ b/README.rst @@ -79,6 +79,8 @@ in development This is the last major release to support Python 3.7. * Drop support for Python 3.6 +* `ExtRegistrar.register_post_processor()` now supports a `priority` argument that is an int. + Highest priority callables will be called first during post-processing. pydoctor 23.9.0 ^^^^^^^^^^^^^^^ diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index c9249dae5..218bb5da9 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -1291,3 +1291,4 @@ def parseDocformat(node: ast.Assign, mod: model.Module) -> None: def setup_pydoctor_extension(r:extensions.ExtRegistrar) -> None: r.register_astbuilder_visitor(TypeAliasVisitorExt) + r.register_post_processor(model.defaultPostProcess, priority=200) diff --git a/pydoctor/extensions/__init__.py b/pydoctor/extensions/__init__.py index 733eab0f4..83965d838 100644 --- a/pydoctor/extensions/__init__.py +++ b/pydoctor/extensions/__init__.py @@ -7,7 +7,7 @@ import importlib import sys -from typing import Any, Callable, Dict, Iterable, Iterator, List, Type, Union, TYPE_CHECKING, cast +from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Type, Union, TYPE_CHECKING, cast # In newer Python versions, use importlib.resources from the standard library. # On older versions, a compatibility package must be installed from PyPI. @@ -113,6 +113,59 @@ def _get_mixins(*mixins: Type[MixinT]) -> Dict[str, List[Type[MixinT]]]: assert False, f"Invalid mixin {mixin.__name__!r}. Mixins must subclass one of the base class." return mixins_by_name +# Largely inspired by docutils Transformer class. +DEFAULT_PRIORITY = 100 +class PriorityProcessor: + """ + Stores L{Callable} and applies them to the system based on priority or insertion order. + The default priority is C{100}, see code source of L{astbuilder.setup_pydoctor_extension}, + and others C{setup_pydoctor_extension} functions. + + Highest priority callables will be called first, when priority is the same it's FIFO order. + + One L{PriorityProcessor} should only be run once on the system. + """ + + def __init__(self, system:'model.System'): + self.system = system + self.applied: List[Callable[['model.System'], None]] = [] + self._post_processors: List[Tuple[object, Callable[['model.System'], None]]] = [] + self._counter = 256 + """Internal counter to keep track of the add order of callables.""" + + def add_post_processor(self, post_processor:Callable[['model.System'], None], + priority:Optional[int]) -> None: + if priority is None: + priority = DEFAULT_PRIORITY + priority_key = self._get_priority_key(priority) + self._post_processors.append((priority_key, post_processor)) + + def _get_priority_key(self, priority:int) -> object: + """ + Return a tuple, `priority` combined with `self._counter`. + + This ensures FIFO order on callables with identical priority. + """ + self._counter -= 1 + return (priority, self._counter) + + def apply_processors(self) -> None: + """Apply all of the stored processors, in priority order.""" + if self.applied: + # this is typically only reached in tests, when we + # call fromText() several times with the same + # system or when we manually call System.postProcess() + self.system.msg('post processing', + 'warning: multiple post-processing pass detected', + thresh=-1) + self.applied.clear() + + self._post_processors.sort() + for p in reversed(self._post_processors): + _, post_processor = p + post_processor(self.system) + self.applied.append(post_processor) + @attr.s(auto_attribs=True) class ExtRegistrar: """ @@ -135,14 +188,18 @@ def register_astbuilder_visitor(self, self.system._astbuilder_visitors.extend(visitor) def register_post_processor(self, - *post_processor: Callable[['model.System'], None]) -> None: + *post_processor: Callable[['model.System'], None], + priority:Optional[int]=None) -> None: """ Register post processor(s). A post-processor is simply a one-argument callable receiving the processed L{model.System} and doing stuff on the L{model.Documentable} tree. + + @param priority: See L{PriorityProcessor}. """ - self.system._post_processors.extend(post_processor) + for p in post_processor: + self.system._post_processor.add_post_processor(p, priority) def load_extension_module(system:'model.System', mod: str) -> None: """ diff --git a/pydoctor/model.py b/pydoctor/model.py index 1e6f1ae28..61f84a2bf 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -21,7 +21,7 @@ from inspect import signature, Signature from pathlib import Path from typing import ( - TYPE_CHECKING, Any, Callable, Collection, Dict, Iterator, List, Mapping, + TYPE_CHECKING, Any, Collection, Dict, Iterator, List, Mapping, Optional, Sequence, Set, Tuple, Type, TypeVar, Union, cast, overload ) from urllib.parse import quote @@ -963,7 +963,7 @@ def __init__(self, options: Optional['Options'] = None): # Initialize the extension system self._factory = factory.Factory() self._astbuilder_visitors: List[Type['astutils.NodeVisitorExt']] = [] - self._post_processors: List[Callable[['System'], None]] = [] + self._post_processor = extensions.PriorityProcessor(self) if self.extensions == _default_extensions: self.extensions = list(extensions.get_extensions()) @@ -1436,29 +1436,10 @@ def postProcess(self) -> None: Analysis of relations between documentables can be done here, without the risk of drawing incorrect conclusions because modules were not fully processed yet. - """ - for cls in self.objectsOfType(Class): - - # Initiate the MROs - cls._init_mro() - # Lookup of constructors - cls._init_constructors() - - # Compute subclasses - for b in cls.baseobjects: - if b is not None: - b.subclasses.append(cls) - - # Checking whether the class is an exception - if is_exception(cls): - cls.kind = DocumentableKind.EXCEPTION - - for attrib in self.objectsOfType(Attribute): - _inherits_instance_variable_kind(attrib) - - for post_processor in self._post_processors: - post_processor(self) + @See: L{extensions.PriorityProcessor}. + """ + self._post_processor.apply_processors() def fetchIntersphinxInventories(self, cache: CacheT) -> None: """ @@ -1467,6 +1448,25 @@ def fetchIntersphinxInventories(self, cache: CacheT) -> None: for url in self.options.intersphinx: self.intersphinx.update(cache, url) +def defaultPostProcess(system:'System') -> None: + for cls in system.objectsOfType(Class): + # Initiate the MROs + cls._init_mro() + # Lookup of constructors + cls._init_constructors() + + # Compute subclasses + for b in cls.baseobjects: + if b is not None: + b.subclasses.append(cls) + + # Checking whether the class is an exception + if is_exception(cls): + cls.kind = DocumentableKind.EXCEPTION + + for attrib in system.objectsOfType(Attribute): + _inherits_instance_variable_kind(attrib) + def _inherits_instance_variable_kind(attr: Attribute) -> None: """ If any of the inherited members of a class variable is an instance variable, diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index b5af2de2e..b32a474f7 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -381,9 +381,9 @@ def test_relative_import_past_top( ''', modname='mod', parent_name='pkg', system=system) captured = capsys.readouterr().out if level == 1: - assert not captured + assert 'relative import level' not in captured else: - assert f'pkg.mod:2: relative import level ({level}) too high\n' == captured + assert f'pkg.mod:2: relative import level ({level}) too high\n' in captured @systemcls_param def test_class_with_base_from_module(systemcls: Type[model.System]) -> None: diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index 4d4a8b521..a8a495018 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -1482,6 +1482,9 @@ def test_module_docformat(capsys: CapSys) -> None: captured = capsys.readouterr().out assert not captured + system = model.System() + system.options.docformat = 'epytext' + mod = fromText(''' """ Link to pydoctor: `pydoctor `_. @@ -1520,14 +1523,18 @@ def f(a: str, b: int): system = model.System() system.options.docformat = 'restructuredtext' - top = fromText(top_src, modname='top', is_package=True, system=system) - fromText(pkg_src, modname='pkg', parent_name='top', is_package=True, - system=system) - mod = fromText(mod_src, modname='top.pkg.mod', parent_name='top.pkg', system=system) + builder = system.systemBuilder(system) + builder.addModuleString(top_src, modname='top', is_package=True) + builder.addModuleString(pkg_src, modname='pkg', parent_name='top', is_package=True) + builder.addModuleString(mod_src, modname='mod', parent_name='top.pkg') + builder.buildModules() + top = system.allobjects['top'] + mod = system.allobjects['top.pkg.mod'] + assert isinstance(mod, model.Module) + assert mod.docformat == 'epytext' captured = capsys.readouterr().out assert not captured - assert ''.join(docstring2html(top.contents['f']).splitlines()) == ''.join(docstring2html(mod.contents['f']).splitlines()) @@ -1553,10 +1560,14 @@ def f(self, a: str, b: int): ''' system = model.System() + builder = system.systemBuilder(system) system.options.docformat = 'epytext' - mod = fromText(mod_src, modname='mod', system=system) - mod2 = fromText(mod2_src, modname='mod2', system=system) + builder.addModuleString(mod_src, modname='mod',) + builder.addModuleString(mod2_src, modname='mod2',) + builder.buildModules() + mod = system.allobjects['mod'] + mod2 = system.allobjects['mod2'] captured = capsys.readouterr().out assert not captured @@ -1610,11 +1621,14 @@ def f(a, b): ''' system = model.System() + builder = system.systemBuilder(system) system.options.docformat = 'restructuredtext' - fromText("", modname='pack', system=system, is_package=True) - fromText(mod1, modname='mod1', system=system, parent_name='pack') - mod = fromText(mod2, modname='mod2', system=system, parent_name='pack') + builder.addModuleString("", modname='pack', is_package=True) + builder.addModuleString(mod1, modname='mod1',parent_name='pack') + builder.addModuleString(mod2, modname='mod2', parent_name='pack') + builder.buildModules() + mod = system.allobjects['pack.mod2'] captured = capsys.readouterr().out assert not captured diff --git a/pydoctor/test/test_model.py b/pydoctor/test/test_model.py index 24e198f81..9dc9d4f33 100644 --- a/pydoctor/test/test_model.py +++ b/pydoctor/test/test_model.py @@ -13,7 +13,7 @@ from twisted.web.template import Tag from pydoctor.options import Options -from pydoctor import model, stanutils +from pydoctor import model, stanutils, extensions from pydoctor.templatewriter import pages from pydoctor.utils import parse_privacy_tuple from pydoctor.sphinx import CacheT @@ -548,3 +548,28 @@ def f():... assert not innerFn.isNameDefined('F') assert not innerFn.isNameDefined('var') assert innerFn.isNameDefined('f') + +def test_priority_processor(capsys:CapSys) -> None: + system = model.System() + r = extensions.ExtRegistrar(system) + processor = system._post_processor + processor._post_processors.clear() + + r.register_post_processor(lambda s:print('priority 200'), priority=200) + r.register_post_processor(lambda s:print('priority 100')) + r.register_post_processor(lambda s:print('priority 25'), priority=25) + r.register_post_processor(lambda s:print('priority 150'), priority=150) + r.register_post_processor(lambda s:print('priority 100 (bis)')) + r.register_post_processor(lambda s:print('priority 200 (bis)'), priority=200) + + assert len(processor._post_processors)==6 + processor.apply_processors() + assert len(processor.applied)==6 + + assert capsys.readouterr().out.strip().splitlines() == ['priority 200', + 'priority 200 (bis)', + 'priority 150', + 'priority 100', + 'priority 100 (bis)', + 'priority 25', + ]