From b62d2a7c9388646e79f744752f6e0d7eb3d56862 Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Thu, 8 Jun 2023 18:45:47 +0200 Subject: [PATCH 1/9] Document return codes (#707) * Document return codes. Fix #540 --- docs/source/quickstart.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index f3a69cbdb..5c128b233 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -54,3 +54,15 @@ Output files are static HTML pages which require no extra server-side support. Here is a `GitHub Action example `_ to automatically publish your API documentation to your default GitHub Pages website. + +Return codes +------------ + +Pydoctor is a pretty verbose tool by default. It’s quite unlikely that you get a zero exit code on the first run. +But don’t worry, pydoctor should have produced useful HTML pages no matter your project design or docstrings. + +Exit codes includes: +- ``0``: All docstrings are well formatted (warnings may be printed). +- ``1``: Pydoctor crashed with traceback (default Python behaviour). +- ``2``: Some docstrings are mal formatted. +- ``3``: Pydoctor detects some warnings and ``--warnings-as-errors`` is enabled. From 562328e72f6c228915bd83f976c6711ab5d9d85f Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Thu, 8 Jun 2023 18:46:14 +0200 Subject: [PATCH 2/9] Add a documentation section on profiling (#717) * Add a section on profiling --- docs/source/contrib.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/source/contrib.rst b/docs/source/contrib.rst index 2d4156e1c..560b3c973 100644 --- a/docs/source/contrib.rst +++ b/docs/source/contrib.rst @@ -123,6 +123,32 @@ Such new packages shouldn't get vendored. They need to be packaged in Debian. Best is to get in contact with the DPT to talk about about new requirements and the best way to get things done. +Profiling pydoctor with austin and speedscope +--------------------------------------------- + +1. Install austin (https://github.com/P403n1x87/austin) +2. Install austin-python (https://pypi.org/project/austin-python/) +3. Run program under austin + + .. code:: + + $ sudo austin -i 1ms -C -o pydoctor.austin pydoctor + +4. Convert .austin to .speedscope (austin2speedscope comes from austin-python) + + .. code:: + + $ austin2speedscope pydoctor.austin pydoctor.speedscope + + +5. Open https://speedscope.app and load pydoctor.speedscope into it. + +Note on sampling interval +~~~~~~~~~~~~~~~~~~~~~~~~~ + +On our large repo I turn down the sampling interval from 100us to 1ms to make +the resulting ``.speedscope`` file a manageable size (15MB instead of 158MB which is too large to put into a gist.) + Author Design Notes ------------------- From b991cc96c362108a9b791fd9e5662276720c643c Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Thu, 8 Jun 2023 19:38:43 +0200 Subject: [PATCH 3/9] Recognize variadic generics type variables (PEP 646). (#708) Fixes #629 --- README.rst | 1 + pydoctor/astbuilder.py | 6 +++++- pydoctor/test/test_astbuilder.py | 13 +++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b1025fef4..681a1f871 100644 --- a/README.rst +++ b/README.rst @@ -80,6 +80,7 @@ in development scope when possible, when impossible, the theoretical runtime scopes are used. A warning can be reported when an annotation name is ambiguous (can be resolved to different names depending on the scope context) with option ``-v``. +* Recognize variadic generics type variables (PEP 646). pydoctor 23.4.1 ^^^^^^^^^^^^^^^ diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index b357b04f9..39f77dfdf 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -79,7 +79,11 @@ class TypeAliasVisitorExt(extensions.ModuleVisitorExt): """ def _isTypeVariable(self, ob: model.Attribute) -> bool: if ob.value is not None: - if isinstance(ob.value, ast.Call) and node2fullname(ob.value.func, ob) in ('typing.TypeVar', 'typing_extensions.TypeVar'): + if isinstance(ob.value, ast.Call) and \ + node2fullname(ob.value.func, ob) in ('typing.TypeVar', + 'typing_extensions.TypeVar', + 'typing.TypeVarTuple', + 'typing_extensions.TypeVarTuple'): return True return False diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index e30c1b3a9..baae2a8e0 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -2119,6 +2119,19 @@ def __init__(self): assert mod.contents['F'].contents['Pouet'].kind == model.DocumentableKind.INSTANCE_VARIABLE assert mod.contents['F'].contents['Q'].kind == model.DocumentableKind.INSTANCE_VARIABLE +@systemcls_param +def test_typevartuple(systemcls: Type[model.System]) -> None: + """ + Variadic type variables are recognized. + """ + + mod = fromText(''' + from typing import TypeVarTuple + Shape = TypeVarTuple('Shape') + ''', systemcls=systemcls) + + assert mod.contents['Shape'].kind == model.DocumentableKind.TYPE_VARIABLE + @systemcls_param def test_prepend_package(systemcls: Type[model.System]) -> None: """ From 39a34ca93bfb2822a8106f45f33d490463a17880 Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Fri, 9 Jun 2023 14:00:07 +0200 Subject: [PATCH 4/9] Special case **kwargs in parameter table (#698) * Fixes #697 * Add changelog entry * Add a test to ensure the **kwargs are presented last in the parameter table. * Make sure **kwargs row is presented last in the parameter table * Fix typos * Improve error message when var-keyword arguments are wrongly documented. * Simplification * Refactor tests --------- Co-authored-by: tristanlatr --- README.rst | 2 + pydoctor/epydoc2stan.py | 70 ++++++++++++++----- pydoctor/test/test_epydoc2stan.py | 108 ++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index 681a1f871..eaafbd3aa 100644 --- a/README.rst +++ b/README.rst @@ -76,6 +76,8 @@ What's New? in development ^^^^^^^^^^^^^^ +* Do not show `**kwargs` when keywords are specifically documented with the `keyword` field + and no specific documentation is given for the `**kwargs` entry. * Fix annotation resolution edge cases: names are resolved in the context of the module scope when possible, when impossible, the theoretical runtime scopes are used. A warning can be reported when an annotation name is ambiguous (can be resolved to different names diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index 62dc89468..fa34e94be 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -6,7 +6,7 @@ import enum from typing import ( TYPE_CHECKING, Any, Callable, ClassVar, DefaultDict, Dict, Generator, - Iterator, List, Mapping, Optional, Sequence, Tuple, + Iterator, List, Mapping, Optional, Sequence, Tuple, Union, ) import ast import re @@ -99,9 +99,20 @@ def format(self) -> Generator[Tag, None, None]: yield tags.td(formatted, colspan="2") @attr.s(auto_attribs=True) -class SignatureDesc(FieldDesc): +class _SignatureDesc(FieldDesc): type_origin: Optional['FieldOrigin'] = None + def is_documented(self) -> bool: + return bool(self.body or self.type_origin is FieldOrigin.FROM_DOCSTRING) + +@attr.s(auto_attribs=True) +class ReturnDesc(_SignatureDesc):... + +@attr.s(auto_attribs=True) +class ParamDesc(_SignatureDesc):... + +@attr.s(auto_attribs=True) +class KeywordDesc(_SignatureDesc):... class RaisesDesc(FieldDesc): """Description of an exception that can be raised by function/method.""" @@ -243,8 +254,8 @@ def __init__(self, obj: model.Documentable): self.types: Dict[str, Optional[ParamType]] = {} - self.parameter_descs: List[SignatureDesc] = [] - self.return_desc: Optional[SignatureDesc] = None + self.parameter_descs: List[Union[ParamDesc, KeywordDesc]] = [] + self.return_desc: Optional[ReturnDesc] = None self.yields_desc: Optional[FieldDesc] = None self.raise_descs: List[RaisesDesc] = [] self.warns_desc: List[FieldDesc] = [] @@ -277,7 +288,7 @@ def set_param_types_from_annotations( ann_ret = annotations['return'] assert ann_ret is not None # ret_type would be None otherwise if not is_none_literal(ann_ret): - self.return_desc = SignatureDesc(type=ret_type.stan, type_origin=ret_type.origin) + self.return_desc = ReturnDesc(type=ret_type.stan, type_origin=ret_type.origin) @staticmethod def _report_unexpected_argument(field:Field) -> None: @@ -287,7 +298,7 @@ def _report_unexpected_argument(field:Field) -> None: def handle_return(self, field: Field) -> None: self._report_unexpected_argument(field) if not self.return_desc: - self.return_desc = SignatureDesc() + self.return_desc = ReturnDesc() self.return_desc.body = field.format() handle_returns = handle_return @@ -301,7 +312,7 @@ def handle_yield(self, field: Field) -> None: def handle_returntype(self, field: Field) -> None: self._report_unexpected_argument(field) if not self.return_desc: - self.return_desc = SignatureDesc() + self.return_desc = ReturnDesc() self.return_desc.type = field.format() self.return_desc.type_origin = FieldOrigin.FROM_DOCSTRING handle_rtype = handle_returntype @@ -314,6 +325,10 @@ def handle_yieldtype(self, field: Field) -> None: handle_ytype = handle_yieldtype def _handle_param_name(self, field: Field) -> Optional[str]: + """ + Returns the Field name and trigger a few warnings for a few scenarios. + Note that the return type could be L{VariableArgument} or L{KeywordArgument} or L{str}. + """ name = field.arg if name is None: field.report('Parameter name missing') @@ -327,7 +342,7 @@ def _handle_param_name(self, field: Field) -> Optional[str]: # Constructor parameters can be documented on the class. annotations = field.source.constructor_params # This might look useless, but it's needed in order to keep the - # right str type: str, VariableArgument or KeyowrdArgument. And then add the stars accordingly. + # right return type and then add the stars accordingly. if annotations is not None: for param_name, _ in annotations.items(): if param_name == name: @@ -352,7 +367,14 @@ def _handle_param_not_found(self, name: str, field: Field) -> None: if name in source.constructor_params: # Constructor parameters can be documented on the class. return - field.report('Documented parameter "%s" does not exist' % (name,)) + msg = f'Documented parameter "{name}" does not exist' + if any(isinstance(n, KeywordArgument) for n in self.types): + msg += ', variable keywords should be documented with the ' + if _get_docformat(self.obj) in ('google', 'numpy'): + msg += '"Keyword Arguments" section' + else: + msg += '"keyword" field' + field.report(msg) def handle_type(self, field: Field) -> None: if isinstance(self.obj, model.Attribute): @@ -383,7 +405,7 @@ def handle_param(self, field: Field) -> None: if name is not None: if any(desc.name == name for desc in self.parameter_descs): field.report('Parameter "%s" was already documented' % (name,)) - self.parameter_descs.append(SignatureDesc(name=name, body=field.format())) + self.parameter_descs.append(ParamDesc(name=name, body=field.format())) if name not in self.types: self._handle_param_not_found(name, field) @@ -393,7 +415,7 @@ def handle_keyword(self, field: Field) -> None: name = self._handle_param_name(field) if name is not None: # TODO: How should this be matched to the type annotation? - self.parameter_descs.append(SignatureDesc(name=name, body=field.format())) + self.parameter_descs.append(KeywordDesc(name=name, body=field.format())) if name in self.types: field.report('Parameter "%s" is documented as keyword' % (name,)) @@ -450,14 +472,14 @@ def handle(self, field: Field) -> None: m(field) def resolve_types(self) -> None: - """Merge information from 'param' fields and AST analysis.""" + """Merge information from 'param'/'keyword' fields and AST analysis.""" params = {param.name: param for param in self.parameter_descs} any_info = bool(params) # We create a new parameter_descs list to ensure the parameter order # matches the AST order. - new_parameter_descs: List[SignatureDesc] = [] + new_parameter_descs: List[Union[ParamDesc, KeywordDesc]] = [] for index, (name, param_type) in enumerate(self.types.items()): try: param = params.pop(name) @@ -471,7 +493,7 @@ def resolve_types(self) -> None: if name=='cls' and self.obj.kind is model.DocumentableKind.CLASS_METHOD: continue - param = SignatureDesc(name=name, + param = ParamDesc(name=name, type=param_type.stan if param_type else None, type_origin=param_type.origin if param_type else None,) @@ -491,16 +513,32 @@ def resolve_types(self) -> None: if any_info: self.parameter_descs = new_parameter_descs + # loops thought the parameters and remove eventual **kwargs + # entry if keywords are specifically documented. + kwargs = None + has_keywords = False + for p in self.parameter_descs: + if isinstance(p.name, KeywordArgument): + kwargs = p + continue + if isinstance(p, KeywordDesc): + has_keywords = True + if kwargs: + self.parameter_descs.remove(kwargs) + if not has_keywords or kwargs.is_documented(): + # make sure **kwargs row is presented last in the parameter table + self.parameter_descs.append(kwargs) + def format(self) -> Tag: r: List[Tag] = [] # Only include parameter or return sections if any are documented or any type are documented from @type fields. include_params = False - if any((p.body or p.type_origin is FieldOrigin.FROM_DOCSTRING) for p in self.parameter_descs): + if any(p.is_documented() for p in self.parameter_descs): r += format_desc_list('Parameters', self.parameter_descs) include_params = True - if self.return_desc and (include_params or self.return_desc.body or self.return_desc.type_origin is FieldOrigin.FROM_DOCSTRING): + if self.return_desc and (include_params or self.return_desc.is_documented()): r += format_desc_list('Returns', [self.return_desc]) if self.yields_desc: diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index 1e26513ed..4d4a8b521 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -696,6 +696,114 @@ def f(args, kwargs, *a, **kwa) -> None: captured = capsys.readouterr().out assert not captured +def test_func_starargs_hidden_when_keywords_documented(capsys:CapSys) -> None: + """ + When a function accept variable keywords (**kwargs) and keywords are specifically + documented and the **kwargs IS NOT documented: entry for **kwargs IS NOT presented at all. + + In other words: They variable keywords argument documentation is optional when specific documentation + is given for each keyword, and when missing, no warning is raised. + """ + # tests for issue https://github.com/twisted/pydoctor/issues/697 + + mod = fromText(''' + __docformat__='restructuredtext' + def f(one, two, **kwa) -> None: + """ + var-keyword arguments are specifically documented. + + :param one: some regular argument + :param two: some regular argument + :keyword something: An argument + :keyword another: Another + """ + ''') + + html = docstring2html(mod.contents['f']) + assert '**kwa' not in html + assert not capsys.readouterr().out + +def test_func_starargs_shown_when_documented(capsys:CapSys) -> None: + """ + When a function accept variable keywords (**kwargs) and keywords are specifically + documented and the **kwargs IS documented: entry for **kwargs IS presented AFTER all keywords. + + In other words: When a function has the keywords arguments, the keywords can have dedicated + docstring, besides the separate documentation for each keyword. + """ + + mod = fromText(''' + __docformat__='restructuredtext' + def f(one, two, **kwa) -> None: + """ + var-keyword arguments are specifically documented as well as other extra keywords. + + :param one: some regular argument + :param two: some regular argument + :param kwa: Other keywords are passed to ``parse`` function. + :keyword something: An argument + :keyword another: Another + """ + ''') + html = docstring2html(mod.contents['f']) + # **kwa should be presented AFTER all other parameters + assert re.match('.+one.+two.+something.+another.+kwa', html, flags=re.DOTALL) + assert not capsys.readouterr().out + +def test_func_starargs_shown_when_undocumented(capsys:CapSys) -> None: + """ + When a function accept variable keywords (**kwargs) and NO keywords are specifically + documented and the **kwargs IS NOT documented: entry for **kwargs IS presented as undocumented. + """ + + mod = fromText(''' + __docformat__='restructuredtext' + def f(one, two, **kwa) -> None: + """ + var-keyword arguments are not specifically documented + + :param one: some regular argument + :param two: some regular argument + """ + ''') + + html = docstring2html(mod.contents['f']) + assert re.match('.+one.+two.+kwa', html, flags=re.DOTALL) + assert not capsys.readouterr().out + +def test_func_starargs_wrongly_documented(capsys: CapSys) -> None: + numpy_wrong = fromText(''' + __docformat__='numpy' + def f(one, **kwargs): + """ + var-keyword arguments are wrongly documented with the "Arguments" section. + + Arguments + --------- + kwargs: + var-keyword arguments + stuff: + a var-keyword argument + """ + ''', modname='numpy_wrong') + + rst_wrong = fromText(''' + __docformat__='restructuredtext' + def f(one, **kwargs): + """ + var-keyword arguments are wrongly documented with the "param" field. + + :param kwargs: var-keyword arguments + :param stuff: a var-keyword argument + """ + ''', modname='rst_wrong') + + docstring2html(numpy_wrong.contents['f']) + assert 'Documented parameter "stuff" does not exist, variable keywords should be documented with the "Keyword Arguments" section' in capsys.readouterr().out + + docstring2html(rst_wrong.contents['f']) + assert 'Documented parameter "stuff" does not exist, variable keywords should be documented with the "keyword" field' in capsys.readouterr().out + def test_summary() -> None: mod = fromText(''' def single_line_summary(): From 14039897c1878d95c55f832666456de124d07c50 Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Sat, 10 Jun 2023 10:47:25 +0200 Subject: [PATCH 5/9] Use Sequence instead of List to annotate the Class._mro attribute (#706) * Fix #638 --- pydoctor/model.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pydoctor/model.py b/pydoctor/model.py index 9180c87b7..8d0d6c2ba 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -531,7 +531,7 @@ def is_exception(cls: 'Class') -> bool: return True return False -def compute_mro(cls:'Class') -> List[Union['Class', str]]: +def compute_mro(cls:'Class') -> Sequence[Union['Class', str]]: """ Compute the method resolution order for this class. This function will also set the @@ -614,7 +614,7 @@ class Class(CanContainImportsDocumentable): # set in post-processing: _finalbaseobjects: Optional[List[Optional['Class']]] = None _finalbases: Optional[List[str]] = None - _mro: Optional[List[Union['Class', str]]] = None + _mro: Optional[Sequence[Union['Class', str]]] = None def setup(self) -> None: super().setup() @@ -679,10 +679,10 @@ def _init_constructors(self) -> None: epydoc2stan.populate_constructors_extra_info(self) @overload - def mro(self, include_external:'Literal[True]', include_self:bool=True) -> List[Union['Class', str]]:... + def mro(self, include_external:'Literal[True]', include_self:bool=True) -> Sequence[Union['Class', str]]:... @overload - def mro(self, include_external:'Literal[False]'=False, include_self:bool=True) -> List['Class']:... - def mro(self, include_external:bool=False, include_self:bool=True) -> List[Union['Class', str]]: # type:ignore[misc] + def mro(self, include_external:'Literal[False]'=False, include_self:bool=True) -> Sequence['Class']:... + def mro(self, include_external:bool=False, include_self:bool=True) -> Sequence[Union['Class', str]]: """ Get the method resution order of this class. @@ -691,8 +691,7 @@ def mro(self, include_external:bool=False, include_self:bool=True) -> List[Union """ if self._mro is None: return list(self.allbases(include_self)) - - _mro: List[Union[str, Class]] + _mro: Sequence[Union[str, Class]] if include_external is False: _mro = [o for o in self._mro if not isinstance(o, str)] else: From 301d239bdba280e4fb4d986228b3d6a9944eb25d Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Mon, 12 Jun 2023 11:17:24 +0200 Subject: [PATCH 6/9] Link top level names in class hierarchy with intersphinx when possible. (#709) * Takle part of issue #310. Link to names with intersphinx when possible. * Add highlighting when clicking on "View In Hierarchy" link from class page --- README.rst | 2 ++ pydoctor/linker.py | 18 +++++++-------- pydoctor/templatewriter/summary.py | 33 +++++++++++++++++++--------- pydoctor/test/test_templatewriter.py | 16 ++++++++++++-- pydoctor/themes/base/apidocs.css | 5 +++++ 5 files changed, 53 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index eaafbd3aa..c9611e55c 100644 --- a/README.rst +++ b/README.rst @@ -82,6 +82,8 @@ in development scope when possible, when impossible, the theoretical runtime scopes are used. A warning can be reported when an annotation name is ambiguous (can be resolved to different names depending on the scope context) with option ``-v``. +* Improve the class hierarchy such that it links top level names with intersphinx when possible. +* Add highlighting when clicking on "View In Hierarchy" link from class page. * Recognize variadic generics type variables (PEP 646). pydoctor 23.4.1 diff --git a/pydoctor/linker.py b/pydoctor/linker.py index 9bb4c08cc..c2430a4b2 100644 --- a/pydoctor/linker.py +++ b/pydoctor/linker.py @@ -45,6 +45,13 @@ def taglink(o: 'model.Documentable', page_url: str, ret(title=o.fullName()) return ret +def intersphinx_link(label:"Flattenable", url:str) -> Tag: + """ + Create a intersphinx link. + + It's special because it uses the 'intersphinx-link' CSS class. + """ + return tags.a(label, href=url, class_='intersphinx-link') class _EpydocLinker(DocstringLinker): """ @@ -92,13 +99,6 @@ def switch_context(self, ob:Optional['model.Documentable']) -> Iterator[None]: self._page_object = old_page_object self.reporting_obj = old_reporting_object - @staticmethod - def _create_intersphinx_link(label:"Flattenable", url:str) -> Tag: - """ - Create a link with the special 'intersphinx-link' CSS class. - """ - return tags.a(label, href=url, class_='intersphinx-link') - def look_for_name(self, name: str, candidates: Iterable['model.Documentable'], @@ -139,7 +139,7 @@ def link_to(self, identifier: str, label: "Flattenable") -> Tag: url = self.look_for_intersphinx(fullID) if url is not None: - return self._create_intersphinx_link(label, url=url) + return intersphinx_link(label, url=url) link = tags.transparent(label) return link @@ -152,7 +152,7 @@ def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag: xref = label else: if isinstance(resolved, str): - xref = self._create_intersphinx_link(label, url=resolved) + xref = intersphinx_link(label, url=resolved) else: xref = taglink(resolved, self.page_url, label) diff --git a/pydoctor/templatewriter/summary.py b/pydoctor/templatewriter/summary.py index f10c46f6d..0aeea6c8f 100644 --- a/pydoctor/templatewriter/summary.py +++ b/pydoctor/templatewriter/summary.py @@ -8,7 +8,7 @@ from twisted.web.template import Element, Tag, TagLoader, renderer, tags -from pydoctor import epydoc2stan, model +from pydoctor import epydoc2stan, model, linker from pydoctor.templatewriter import TemplateLookup from pydoctor.templatewriter.pages import Page, objects_order @@ -18,7 +18,7 @@ def moduleSummary(module: model.Module, page_url: str) -> Tag: r: Tag = tags.li( - tags.code(epydoc2stan.taglink(module, page_url, label=module.name)), ' - ', + tags.code(linker.taglink(module, page_url, label=module.name)), ' - ', epydoc2stan.format_summary(module) ) if module.isPrivate: @@ -37,7 +37,7 @@ def moduleSummary(module: model.Module, page_url: str) -> Tag: li = tags.li(class_='compact-modules') for m in sorted(contents, key=objects_order): span = tags.span() - span(tags.code(epydoc2stan.taglink(m, m.url, label=m.name))) + span(tags.code(linker.taglink(m, m.url, label=m.name))) span(', ') if m.isPrivate: span(class_='private') @@ -91,7 +91,7 @@ def findRootClasses( for name, base in zip(cls.bases, cls.baseobjects): if base is None or not base.isVisible: # The base object is in an external library or filtered out (not visible) - # Take special care to avoid AttributeError: 'ZopeInterfaceClass' object has no attribute 'append'. + # Take special care to avoid AttributeError: 'Class' object has no attribute 'append'. if isinstance(roots.get(name), model.Class): roots[name] = [cast(model.Class, roots[name])] cast(List[model.Class], roots.setdefault(name, [])).append(cls) @@ -139,8 +139,8 @@ def subclassesFrom( if name not in anchors: r(tags.a(name=name)) anchors.add(name) - r(tags.code(epydoc2stan.taglink(cls, page_url)), ' - ', - epydoc2stan.format_summary(cls)) + r(tags.div(tags.code(linker.taglink(cls, page_url)), ' - ', + epydoc2stan.format_summary(cls))) scs = [sc for sc in cls.subclasses if sc.system is hostsystem and ' ' not in sc.fullName() and sc.isVisible] if len(scs) > 0: @@ -172,7 +172,20 @@ def stuff(self, request: object, tag: Tag) -> Tag: if isinstance(o, model.Class): t(subclassesFrom(self.system, o, anchors, self.filename)) else: - item = tags.li(tags.code(b)) + url = self.system.intersphinx.getLink(b) + if url: + link:"Flattenable" = linker.intersphinx_link(b, url) + else: + # TODO: we should find a way to use the pyval colorizer instead + # of manually creating the intersphinx link, this would allow to support + # linking to namedtuple(), proxyForInterface() and all other ast constructs. + # But the issue is that we're using the string form of base objects in order + # to compare and aggregate them, as a consequence we can't directly use the colorizer. + # Another side effect is that subclasses of collections.namedtuple() and namedtuple() + # (depending on how the name is imported) will not be aggregated under the same list item :/ + link = b + item = tags.li(tags.code(link)) + if all(isClassNodePrivate(sc) for sc in o): # This is an external class used only by private API; # mark the whole node private. @@ -231,7 +244,7 @@ def link(obj: model.Documentable) -> Tag: if obj.kind: attributes["data-type"] = epydoc2stan.format_kind(obj.kind) return tags.code( - epydoc2stan.taglink(obj, NameIndexPage.filename), **attributes + linker.taglink(obj, NameIndexPage.filename), **attributes ) name2obs: DefaultDict[str, List[model.Documentable]] = defaultdict(list) for obj in self.initials[self.my_letter]: @@ -295,7 +308,7 @@ def roots(self, request: object, tag: Tag) -> "Flattenable": r = [] for o in self.system.rootobjects: r.append(tag.clone().fillSlots(root=tags.code( - epydoc2stan.taglink(o, self.filename) + linker.taglink(o, self.filename) ))) return r @@ -340,7 +353,7 @@ def stuff(self, request: object, tag: Tag) -> Tag: assert kind is not None # 'kind is None' makes the object invisible tag(tags.li( epydoc2stan.format_kind(kind), " - ", - tags.code(epydoc2stan.taglink(o, self.filename)) + tags.code(linker.taglink(o, self.filename)) )) return tag diff --git a/pydoctor/test/test_templatewriter.py b/pydoctor/test/test_templatewriter.py index ed1920282..ed976ce61 100644 --- a/pydoctor/test/test_templatewriter.py +++ b/pydoctor/test/test_templatewriter.py @@ -14,9 +14,10 @@ HtmlTemplate, UnsupportedTemplateVersion, OverrideTemplateNotAllowed) from pydoctor.templatewriter.pages.table import ChildTable -from pydoctor.templatewriter.summary import isClassNodePrivate, isPrivate, moduleSummary +from pydoctor.templatewriter.summary import isClassNodePrivate, isPrivate, moduleSummary, ClassIndexPage from pydoctor.test.test_astbuilder import fromText, systemcls_param from pydoctor.test.test_packages import processPackage, testpackages +from pydoctor.test.test_epydoc2stan import InMemoryInventory from pydoctor.test import CapSys from pydoctor.themes import get_themes @@ -761,7 +762,6 @@ def test_crash_xmlstring_entities_rst(capsys:CapSys, processtypes:bool) -> None: assert re.match('\n'.join(warnings), out) def test_constructor_renders(capsys:CapSys) -> None: - ... src = '''\ class Animal(object): # pydoctor can infer the constructor to be: "Animal(name)" @@ -773,3 +773,15 @@ def __new__(cls, name): html = getHTMLOf(mod.contents['Animal']) assert 'Constructor: ' in html assert 'Animal(name)' in html + +def test_class_hierarchy_links_top_level_names() -> None: + system = model.System() + system.intersphinx = InMemoryInventory() # type:ignore + src = '''\ + from socket import socket + class Stuff(socket): + ... + ''' + mod = fromText(src, system=system) + index = flatten(ClassIndexPage(mod.system, TemplateLookup(template_dir))) + assert 'href="https://docs.python.org/3/library/socket.html#socket.socket"' in index diff --git a/pydoctor/themes/base/apidocs.css b/pydoctor/themes/base/apidocs.css index ed8feee2d..f4f4326c9 100644 --- a/pydoctor/themes/base/apidocs.css +++ b/pydoctor/themes/base/apidocs.css @@ -1127,6 +1127,11 @@ pre.constant-value { padding: .5em; } #childList a:target ~ .functionBody{ box-shadow: -2px -8px 0px 13px rgb(253 255 223); } +/* in class hierarchy */ +#summaryTree a:target ~ div { + background-color: rgb(253, 255, 223); + box-shadow: 0px 0px 0px 7px rgb(253, 255, 223); +} /* deprecations uses a orange text */ .rst-deprecated > .rst-versionmodified{ From c2f3419bac79c26594b2cdcee676ee5431267dfe Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Mon, 12 Jun 2023 11:26:54 +0200 Subject: [PATCH 7/9] A few optimizations (#714) * Use iterators instead of list comprehention to optimize get_all_documents_flattenable(), which is generally the logest page to generate. * Avoid for loop in get_corpus() by using list comprehension. * Use identify check instead of equlaity check in sidebar generation code. * Avoid for loop by using generator expression * Create local copy of functions used in get_all_documents_flattenable() to reduce lookup time. * Use tuple() instead of list() to create a copy of dict items. --- pydoctor/node2stan.py | 2 +- pydoctor/templatewriter/pages/sidebar.py | 14 ++--- pydoctor/templatewriter/search.py | 65 ++++++++++++------------ 3 files changed, 40 insertions(+), 41 deletions(-) diff --git a/pydoctor/node2stan.py b/pydoctor/node2stan.py index dc155cbcc..6acab4e91 100644 --- a/pydoctor/node2stan.py +++ b/pydoctor/node2stan.py @@ -156,7 +156,7 @@ def starttag(self, node: nodes.Node, tagname: str, suffix: str = '\n', **attribu # iterate through attributes one at a time because some # versions of docutils don't case-normalize attributes. for attr_dict in attr_dicts: - for key, val in list(attr_dict.items()): + for key, val in tuple(attr_dict.items()): # Prefix all CSS classes with "rst-"; and prefix all # names with "rst-" to avoid conflicts. if key.lower() in ('class', 'id', 'name'): diff --git a/pydoctor/templatewriter/pages/sidebar.py b/pydoctor/templatewriter/pages/sidebar.py index 46ee75a7a..c5bea1fc8 100644 --- a/pydoctor/templatewriter/pages/sidebar.py +++ b/pydoctor/templatewriter/pages/sidebar.py @@ -70,7 +70,7 @@ def __init__(self, ob: Documentable, documented_ob: Documentable, self.template_lookup = template_lookup # Does this sidebar section represents the object itself ? - self._represents_documented_ob = self.ob == self.documented_ob + self._represents_documented_ob = self.ob is self.documented_ob @renderer def kind(self, request: IRequest, tag: Tag) -> str: @@ -195,7 +195,7 @@ def docstringToc(self, request: IRequest, tag: Tag) -> Union[Tag, str]: # Only show the TOC if visiting the object page itself, in other words, the TOC do dot show up # in the object's parent section or any other subsections except the main one. - if toc and self.documented_ob == self.ob: + if toc and self.documented_ob is self.ob: return tag.fillSlots(titles=toc) else: return "" @@ -288,9 +288,8 @@ def __init__(self, ob: Documentable, @renderer def items(self, request: IRequest, tag: Tag) -> Iterator['ContentItem']: - for child in self.children: - - yield ContentItem( + return ( + ContentItem( loader=TagLoader(tag), ob=self.ob, child=child, @@ -299,6 +298,7 @@ def items(self, request: IRequest, tag: Tag) -> Iterator['ContentItem']: nested_content_loader=self.nested_content_loader, template_lookup=self.template_lookup, level_depth=self._level_depth) + for child in self.children ) class ContentItem(Element): @@ -329,7 +329,7 @@ def class_(self, request: IRequest, tag: Tag) -> str: # But I found it a little bit too colorful. if self.child.isPrivate: class_ += "private" - if self.child == self.documented_ob: + if self.child is self.documented_ob: class_ += " thisobject" return class_ @@ -350,7 +350,7 @@ def expandableItem(self, request: IRequest, tag: Tag) -> Union[str, 'ExpandableI # pass do_not_expand=True also when an object do not have any members, # instead of expanding on an empty div. return ExpandableItem(TagLoader(tag), self.child, self.documented_ob, nested_contents, - do_not_expand=self.child == self.documented_ob or not nested_contents.has_contents) + do_not_expand=self.child is self.documented_ob or not nested_contents.has_contents) else: return "" diff --git a/pydoctor/templatewriter/search.py b/pydoctor/templatewriter/search.py index 1015957e8..a938920af 100644 --- a/pydoctor/templatewriter/search.py +++ b/pydoctor/templatewriter/search.py @@ -3,7 +3,7 @@ """ from pathlib import Path -from typing import Iterable, Iterator, List, Optional, Tuple, Type, Dict, TYPE_CHECKING +from typing import Iterator, List, Optional, Tuple, Type, Dict, TYPE_CHECKING import json import attr @@ -17,22 +17,27 @@ if TYPE_CHECKING: from twisted.web.template import Flattenable -def get_all_documents_flattenable(system: model.System) -> List[Dict[str, "Flattenable"]]: +def get_all_documents_flattenable(system: model.System) -> Iterator[Dict[str, "Flattenable"]]: """ - Get the all data to be writen into ``all-documents.html`` file. + Get a generator for all data to be writen into ``all-documents.html`` file. """ - documents: List[Dict[str, "Flattenable"]] = [dict( - id=ob.fullName(), - name=epydoc2stan.insert_break_points(ob.name), - fullName=epydoc2stan.insert_break_points(ob.fullName()), - kind=epydoc2stan.format_kind(ob.kind) if ob.kind else '', - type=str(ob.__class__.__name__), - summary=epydoc2stan.format_summary(ob), - url=ob.url, - privacy=str(ob.privacyClass.name)) - - for ob in system.allobjects.values() if ob.isVisible] - return documents + # This function accounts for a substantial proportion of pydoctor runtime. + # So it's optimized. + insert_break_points = epydoc2stan.insert_break_points + format_kind = epydoc2stan.format_kind + format_summary = epydoc2stan.format_summary + + return ({ + 'id': ob.fullName(), + 'name': ob.name, + 'fullName': insert_break_points(ob.fullName()), + 'kind': format_kind(ob.kind) if ob.kind else '', + 'type': str(ob.__class__.__name__), + 'summary': format_summary(ob), + 'url': ob.url, + 'privacy': str(ob.privacyClass.name)} + + for ob in system.allobjects.values() if ob.isVisible) class AllDocuments(Page): @@ -42,7 +47,7 @@ def title(self) -> str: return "All Documents" @renderer - def documents(self, request: None, tag: Tag) -> Iterable[Tag]: + def documents(self, request: None, tag: Tag) -> Iterator[Tag]: for doc in get_all_documents_flattenable(self.system): yield tag.clone().fillSlots(**doc) @@ -110,23 +115,17 @@ def format_kind(self, ob:model.Documentable) -> str: return epydoc2stan.format_kind(ob.kind) if ob.kind else '' def get_corpus(self) -> List[Tuple[Dict[str, Optional[str]], Dict[str, int]]]: - - documents: List[Tuple[Dict[str, Optional[str]], Dict[str, int]]] = [] - - for ob in (o for o in self.system.allobjects.values() if o.isVisible): - - documents.append( - ( - { - f:self.format(ob, f) for f in self.fields - }, - { - "boost": self.get_ob_boost(ob) - } - ) - ) - - return documents + return [ + ( + { + f:self.format(ob, f) for f in self.fields + }, + { + "boost": self.get_ob_boost(ob) + } + ) + for ob in (o for o in self.system.allobjects.values() if o.isVisible) + ] def write(self) -> None: From 72b155ff72b70b3574755a372db1060fb0e769a2 Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Mon, 12 Jun 2023 11:34:46 +0200 Subject: [PATCH 8/9] Better understanding of operator precedence in colorizer (#713) * Use parenthesis in colorized AST only if required. The only exception is in the presentation of the power operator and in unsupported nodes. * Actually fix the issue with dict unpacking without the need for enclosing regular dict values inside parenthesis. --- README.rst | 2 + pydoctor/epydoc/markup/_pyval_repr.py | 48 +++++--- pydoctor/test/epydoc/test_pyval_repr.py | 145 ++++++++++++------------ setup.cfg | 1 + 4 files changed, 112 insertions(+), 84 deletions(-) diff --git a/README.rst b/README.rst index c9611e55c..407ddd4ea 100644 --- a/README.rst +++ b/README.rst @@ -82,6 +82,8 @@ in development scope when possible, when impossible, the theoretical runtime scopes are used. A warning can be reported when an annotation name is ambiguous (can be resolved to different names depending on the scope context) with option ``-v``. +* Improve the AST colorizer to output less parenthesis when it's not required. +* Fix colorization of dictionary unpacking. * Improve the class hierarchy such that it links top level names with intersphinx when possible. * Add highlighting when clicking on "View In Hierarchy" link from class page. * Recognize variadic generics type variables (PEP 646). diff --git a/pydoctor/epydoc/markup/_pyval_repr.py b/pydoctor/epydoc/markup/_pyval_repr.py index dd78b9428..7eae1918f 100644 --- a/pydoctor/epydoc/markup/_pyval_repr.py +++ b/pydoctor/epydoc/markup/_pyval_repr.py @@ -127,6 +127,13 @@ def visit(self, node: ast.AST) -> ast.AST: return node # TODO: add support for comparators when needed. +# _OperatorDelimitier is needed for: +# - IfExp +# - UnaryOp +# - BinOp, needs special handling for power operator +# - Compare +# - BoolOp +# - Lambda class _OperatorDelimiter: """ A context manager that can add enclosing delimiters to nested operators when needed. @@ -135,7 +142,7 @@ class _OperatorDelimiter: """ def __init__(self, colorizer: 'PyvalColorizer', state: _ColorizerState, - node: Union[ast.UnaryOp, ast.BinOp, ast.BoolOp]) -> None: + node: Union[ast.UnaryOp, ast.BinOp, ast.BoolOp],) -> None: self.discard = True """No parenthesis by default.""" @@ -148,12 +155,17 @@ def __init__(self, colorizer: 'PyvalColorizer', state: _ColorizerState, # See _Parentage class, applied in PyvalColorizer._colorize_ast() parent_node: Optional[ast.AST] = getattr(node, 'parent', None) - if isinstance(parent_node, (ast.UnaryOp, ast.BinOp, ast.BoolOp)): + if parent_node: precedence = astor.op_util.get_op_precedence(node.op) - parent_precedence = astor.op_util.get_op_precedence(parent_node.op) - # Add parenthesis when precedences are equal to avoid confusions - # and correctly handle the Pow special case without too much annoyance. - if precedence <= parent_precedence: + if isinstance(parent_node, (ast.UnaryOp, ast.BinOp, ast.BoolOp)): + parent_precedence = astor.op_util.get_op_precedence(parent_node.op) + if isinstance(parent_node.op, ast.Pow) or isinstance(parent_node, ast.BoolOp): + parent_precedence+=1 + else: + parent_precedence = colorizer.explicit_precedence.get( + node, astor.op_util.Precedence.highest) + + if precedence < parent_precedence: self.discard = False def __enter__(self) -> '_OperatorDelimiter': @@ -261,6 +273,9 @@ def __init__(self, linelen:Optional[int], maxlines:int, linebreakok:bool=True, r self.maxlines: Union[int, float] = maxlines if maxlines!=0 else float('inf') self.linebreakok = linebreakok self.refmap = refmap if refmap is not None else {} + # some edge cases require to compute the precedence ahead of time and can't be + # easily done with access only to the parent node of some operators. + self.explicit_precedence:Dict[ast.AST, int] = {} #//////////////////////////////////////////////////////////// # Colorization Tags & other constants @@ -294,6 +309,10 @@ def __init__(self, linelen:Optional[int], maxlines:int, linebreakok:bool=True, r RE_COMPILE_SIGNATURE = signature(re.compile) + def _set_precedence(self, precedence:int, *node:ast.AST) -> None: + for n in node: + self.explicit_precedence[n] = precedence + def colorize(self, pyval: Any) -> ColorizedPyvalRepr: """ Entry Point. @@ -351,10 +370,6 @@ def _colorize(self, pyval: Any, state: _ColorizerState) -> None: elif pyvaltype is frozenset: self._multiline(self._colorize_iter, pyval, state, prefix='frozenset([', suffix='])') - elif pyvaltype is dict: - self._multiline(self._colorize_dict, - list(pyval.items()), - state, prefix='{', suffix='}') elif pyvaltype is list: self._multiline(self._colorize_iter, pyval, state, prefix='[', suffix=']') elif issubclass(pyvaltype, ast.AST): @@ -447,15 +462,20 @@ def _colorize_iter(self, pyval: Iterable[Any], state: _ColorizerState, if suffix is not None: self._output(suffix, self.GROUP_TAG, state) - def _colorize_dict(self, items: Iterable[Tuple[Any, Any]], state: _ColorizerState, prefix: str, suffix: str) -> None: + def _colorize_ast_dict(self, items: Iterable[Tuple[Optional[ast.AST], ast.AST]], + state: _ColorizerState, prefix: str, suffix: str) -> None: self._output(prefix, self.GROUP_TAG, state) indent = state.charpos for i, (key, val) in enumerate(items): if i>=1: self._insert_comma(indent, state) state.result.append(self.WORD_BREAK_OPPORTUNITY) - self._colorize(key, state) - self._output(': ', self.COLON_TAG, state) + if key: + self._set_precedence(astor.op_util.Precedence.Comma, val) + self._colorize(key, state) + self._output(': ', self.COLON_TAG, state) + else: + self._output('**', None, state) self._colorize(val, state) self._output(suffix, self.GROUP_TAG, state) @@ -546,7 +566,7 @@ def _colorize_ast(self, pyval: ast.AST, state: _ColorizerState) -> None: self._multiline(self._colorize_iter, pyval.elts, state, prefix='set([', suffix='])') elif isinstance(pyval, ast.Dict): items = list(zip(pyval.keys, pyval.values)) - self._multiline(self._colorize_dict, items, state, prefix='{', suffix='}') + self._multiline(self._colorize_ast_dict, items, state, prefix='{', suffix='}') elif isinstance(pyval, ast.Name): self._colorize_ast_name(pyval, state) elif isinstance(pyval, ast.Attribute): diff --git a/pydoctor/test/epydoc/test_pyval_repr.py b/pydoctor/test/epydoc/test_pyval_repr.py index 0bbb8d342..6dd9fde4f 100644 --- a/pydoctor/test/epydoc/test_pyval_repr.py +++ b/pydoctor/test/epydoc/test_pyval_repr.py @@ -1,4 +1,5 @@ import ast +from functools import partial import sys from textwrap import dedent from typing import Any, Union @@ -518,63 +519,6 @@ def test_tuples_one_value() -> None: ,) """ -def test_dictionaries() -> None: - """Dicts are treated just like lists, except that the ":" is also tagged as - "op".""" - - assert color({'1':33, '2':[1,2,3,{7:'oo'*20}]}) == """ - { - - - ' - - 1 - - ' - : - 33 - , - - - - - ' - - 2 - - ' - : - [ - - 1 - , - - - - 2 - , - - - - 3 - , - - - - { - - 7 - : - - ' - - oooooooooooooooooooooooooooo - - ↵ - - - ...\n""" - def extract_expr(_ast: ast.Module) -> ast.AST: elem = _ast.body[0] assert isinstance(elem, ast.Expr) @@ -719,14 +663,12 @@ def test_operator_precedences() -> None: (1 + 2) * 3 / 4 """)))) == """ ( - ( 1 + 2 ) * 3 - ) / 4\n""" @@ -734,19 +676,17 @@ def test_operator_precedences() -> None: ((1 + 2) * 3) / 4 """)))) == """ ( - ( 1 + 2 ) * 3 - ) / 4\n""" assert color(extract_expr(ast.parse(dedent(""" - (1 + 2) * (3 / 4) + (1 + 2) * 3 / 4 """)))) == """ ( 1 @@ -754,26 +694,20 @@ def test_operator_precedences() -> None: 2 ) * - ( 3 / - 4 - )\n""" + 4\n""" assert color(extract_expr(ast.parse(dedent(""" - (1 + (2 * 3) / 4) - 1 + 1 + 2 * 3 / 4 - 1 """)))) == """ - ( 1 + - ( 2 * 3 - ) / 4 - ) - 1\n""" @@ -908,6 +842,9 @@ def test_ast_list_tuple() -> None: )\n""" def test_ast_dict() -> None: + """ + Dictionnaries are treated just like lists. + """ assert color(extract_expr(ast.parse(dedent(""" {'1':33, '2':[1,2,3,{7:'oo'*20}]} """)))) == """ @@ -1557,3 +1494,71 @@ def test_refmap_explicit() -> None: assert '' in dump assert '' in dump assert '' in dump + +def check_src_roundtrip(src:str, subtests:Any) -> None: + # from cpython/Lib/test/test_unparse.py + with subtests.test(msg="round trip", src=src): + mod = ast.parse(src) + assert len(mod.body)==1 + expr = mod.body[0] + assert isinstance(expr, ast.Expr) + code = color2(expr.value) + assert code==src + +def test_expressions_parens(subtests:Any) -> None: + check_src = partial(check_src_roundtrip, subtests=subtests) + check_src("1<<(10|1)<<1") + check_src("int|float|complex|None") + check_src("1+1") + check_src("1+2/3") + check_src("(1+2)/3") + check_src("(1+2)*3+4*(5+2)") + check_src("(1+2)*3+4*(5+2)**2") + check_src("~x") + check_src("x and y") + check_src("x and y and z") + check_src("x and (y and x)") + check_src("(x and y) and z") + # cpython tests expected '(x**y)**z**q', + # but too much reasonning is needed to obtain this result, + # because the power operator is reassociative... + check_src("(x**y)**(z**q)") + check_src("((x**y)**z)**q") + check_src("x>>y") + check_src("x<>y and x>>z") + check_src("x+y-z*q^t**k") + + check_src("flag&(other|foo)") + + # with astor (which adds a lot of parenthesis :/) + if sys.version_info>=(3,8): + check_src("(a := b)") + if sys.version_info>=(3,7): + check_src("(await x)") + check_src("(x if x else y)") + check_src("(lambda x: x)") + check_src("(lambda : int)()") + check_src("not (x == y)") + check_src("(x == (not y))") + check_src("(P * V if P and V else n * R * T)") + check_src("(lambda P, V, n: P * V == n * R * T)") + + check_src("f(**x)") + check_src("{**x}") + + check_src("(-1)**7") + check_src("(-1.0)**8") + check_src("(-1j)**6") + check_src("not True or False") + check_src("True or not False") + + check_src("(3).__abs__()") + + check_src("f(**([] or 5))") + check_src("{**([] or 5)}") + check_src("{**(~{})}") + check_src("{**(not {})}") + check_src("{**({} == {})}") + check_src("{**{'y': 2}, 'x': 1, None: True}") + check_src("{**{'y': 2}, **{'x': 1}}") diff --git a/setup.cfg b/setup.cfg index 916b0b151..8b10c2cb5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -69,6 +69,7 @@ test = cython-test-exception-raiser==1.0.0 bs4 Sphinx>=3.5 + pytest-subtests [options.entry_points] console_scripts = From 965ed955efde6178cb68f0882a02c34a90204447 Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Tue, 13 Jun 2023 17:26:45 +0200 Subject: [PATCH 9/9] Unstring ALL type aliases (#705) * Fixes #704 --- README.rst | 1 + pydoctor/astbuilder.py | 16 +++++++------- pydoctor/astutils.py | 4 ++-- pydoctor/test/test_astbuilder.py | 20 +++++++++++++++++- pydoctor/test/test_templatewriter.py | 31 ++++++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 407ddd4ea..acf43a21a 100644 --- a/README.rst +++ b/README.rst @@ -82,6 +82,7 @@ in development scope when possible, when impossible, the theoretical runtime scopes are used. A warning can be reported when an annotation name is ambiguous (can be resolved to different names depending on the scope context) with option ``-v``. +* Fix presentation of type aliases in string form. * Improve the AST colorizer to output less parenthesis when it's not required. * Fix colorization of dictionary unpacking. * Improve the class hierarchy such that it links top level names with intersphinx when possible. diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index 39f77dfdf..e1fc18004 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -92,18 +92,11 @@ def _isTypeAlias(self, ob: model.Attribute) -> bool: Return C{True} if the Attribute is a type alias. """ if ob.value is not None: - - if is_using_annotations(ob.annotation, ('typing.TypeAlias', 'typing_extensions.TypeAlias'), ob): - try: - ob.value = unstring_annotation(ob.value, ob) - except SyntaxError as e: - ob.report(f"invalid type alias: {e}") - return False + if is_using_annotations(ob.annotation, ('typing.TypeAlias', + 'typing_extensions.TypeAlias'), ob): return True - if is_typing_annotation(ob.value, ob.parent): return True - return False def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: @@ -117,7 +110,12 @@ def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: return if self._isTypeAlias(attr) is True: attr.kind = model.DocumentableKind.TYPE_ALIAS + # unstring type aliases + attr.value = unstring_annotation( + # this cast() is safe because _isTypeAlias() return True only if value is not None + cast(ast.expr, attr.value), attr, section='type alias') elif self._isTypeVariable(attr) is True: + # TODO: unstring bound argument of type variables attr.kind = model.DocumentableKind.TYPE_VARIABLE visit_AnnAssign = visit_Assign diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 720349b73..5677e5c68 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -195,7 +195,7 @@ def is_none_literal(node: ast.expr) -> bool: """Does this AST node represent the literal constant None?""" return isinstance(node, (ast.Constant, ast.NameConstant)) and node.value is None -def unstring_annotation(node: ast.expr, ctx:'model.Documentable') -> ast.expr: +def unstring_annotation(node: ast.expr, ctx:'model.Documentable', section:str='annotation') -> ast.expr: """Replace all strings in the given expression by parsed versions. @return: The unstringed node. If parsing fails, an error is logged and the original node is returned. @@ -205,7 +205,7 @@ def unstring_annotation(node: ast.expr, ctx:'model.Documentable') -> ast.expr: except SyntaxError as ex: module = ctx.module assert module is not None - module.report(f'syntax error in annotation: {ex}', lineno_offset=node.lineno) + module.report(f'syntax error in {section}: {ex}', lineno_offset=node.lineno, section=section) return node else: assert isinstance(expr, ast.expr), expr diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index baae2a8e0..a12e74307 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -2395,4 +2395,22 @@ def __init__(self): ''' mod = fromText(src, systemcls=systemcls) - assert getConstructorsText(mod.contents['Animal']) == "Animal()" \ No newline at end of file + assert getConstructorsText(mod.contents['Animal']) == "Animal()" + +@systemcls_param +def test_typealias_unstring(systemcls: Type[model.System]) -> None: + """ + The type aliases are unstringed by the astbuilder + """ + + mod = fromText(''' + from typing import Callable + ParserFunction = Callable[[str, List['ParseError']], 'ParsedDocstring'] + ''', modname='pydoctor.epydoc.markup', systemcls=systemcls) + + typealias = mod.contents['ParserFunction'] + assert isinstance(typealias, model.Attribute) + assert typealias.value + with pytest.raises(StopIteration): + # there is not Constant nodes in the type alias anymore + next(n for n in ast.walk(typealias.value) if isinstance(n, ast.Constant)) \ No newline at end of file diff --git a/pydoctor/test/test_templatewriter.py b/pydoctor/test/test_templatewriter.py index ed976ce61..64a331474 100644 --- a/pydoctor/test/test_templatewriter.py +++ b/pydoctor/test/test_templatewriter.py @@ -14,6 +14,7 @@ HtmlTemplate, UnsupportedTemplateVersion, OverrideTemplateNotAllowed) from pydoctor.templatewriter.pages.table import ChildTable +from pydoctor.templatewriter.pages.attributechild import AttributeChild from pydoctor.templatewriter.summary import isClassNodePrivate, isPrivate, moduleSummary, ClassIndexPage from pydoctor.test.test_astbuilder import fromText, systemcls_param from pydoctor.test.test_packages import processPackage, testpackages @@ -56,6 +57,12 @@ def getHTMLOf(ob: model.Documentable) -> str: wr._writeDocsForOne(ob, f) return f.getvalue().decode() +def getHTMLOfAttribute(ob: model.Attribute) -> str: + assert isinstance(ob, model.Attribute) + tlookup = TemplateLookup(template_dir) + stan = AttributeChild(util.DocGetter(), ob, [], + AttributeChild.lookup_loader(tlookup),) + return flatten(stan) def test_sidebar() -> None: src = ''' @@ -774,6 +781,29 @@ def __new__(cls, name): assert 'Constructor: ' in html assert 'Animal(name)' in html +def test_typealias_string_form_linked() -> None: + """ + The type aliases should be unstring before beeing presented to reader, such that + all elements can be linked. + + Test for issue https://github.com/twisted/pydoctor/issues/704 + """ + + mod = fromText(''' + from typing import Callable + ParserFunction = Callable[[str, List['ParseError']], 'ParsedDocstring'] + class ParseError: + ... + class ParsedDocstring: + ... + ''', modname='pydoctor.epydoc.markup') + + typealias = mod.contents['ParserFunction'] + assert isinstance(typealias, model.Attribute) + html = getHTMLOfAttribute(typealias) + assert 'href="pydoctor.epydoc.markup.ParseError.html"' in html + assert 'href="pydoctor.epydoc.markup.ParsedDocstring.html"' in html + def test_class_hierarchy_links_top_level_names() -> None: system = model.System() system.intersphinx = InMemoryInventory() # type:ignore @@ -785,3 +815,4 @@ class Stuff(socket): mod = fromText(src, system=system) index = flatten(ClassIndexPage(mod.system, TemplateLookup(template_dir))) assert 'href="https://docs.python.org/3/library/socket.html#socket.socket"' in index +