From 44a1a39f55afb678aa2d7e02770ea79f3095b50d Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Fri, 8 Sep 2023 16:53:17 -0400 Subject: [PATCH 1/5] Fix CI (#730) * Require Sphinx<7.0.0 for the tests. * Twisted does not support python 3.7 anymore * Fix introspection code for cython3 generated functions * Remove deprecated key in readthedocs.yml config file --- .github/workflows/unit.yaml | 2 +- README.rst | 1 + pydoctor/model.py | 7 +++++-- pydoctor/sphinx_ext/build_apidocs.py | 4 ++-- pydoctor/templatewriter/__init__.py | 5 ++--- pydoctor/test/test_sphinx.py | 2 +- readthedocs.yml | 7 +++++-- setup.cfg | 4 ++-- 8 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml index 751e089e1..eb1fc3c87 100644 --- a/.github/workflows/unit.yaml +++ b/.github/workflows/unit.yaml @@ -46,7 +46,7 @@ jobs: tox -e test - name: Run unit tests with latest Twisted version - if: matrix.python-version != '3.6' && matrix.python-version != 'pypy-3.6' + if: matrix.python-version != '3.7' && matrix.python-version != '3.6' && matrix.python-version != 'pypy-3.6' run: | tox -e test-latest-twisted diff --git a/README.rst b/README.rst index acf43a21a..48a8df2ce 100644 --- a/README.rst +++ b/README.rst @@ -88,6 +88,7 @@ in development * Improve the class hierarchy such that it links top level names with intersphinx when possible. * Add highlighting when clicking on "View In Hierarchy" link from class page. * Recognize variadic generics type variables (PEP 646). +* Fix support for introspection of cython3 generated modules. pydoctor 23.4.1 ^^^^^^^^^^^^^^^ diff --git a/pydoctor/model.py b/pydoctor/model.py index 8d0d6c2ba..310529fe0 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -1257,8 +1257,11 @@ def _introspectThing(self, thing: object, parent: CanContainImportsDocumentable, for k, v in thing.__dict__.items(): if (isinstance(v, func_types) # In PyPy 7.3.1, functions from extensions are not - # instances of the abstract types in func_types - or (hasattr(v, "__class__") and v.__class__.__name__ == 'builtin_function_or_method')): + # instances of the abstract types in func_types, it will have the type 'builtin_function_or_method'. + # Additionnaly cython3 produces function of type 'cython_function_or_method', + # so se use a heuristic on the class name as a fall back detection. + or (hasattr(v, "__class__") and + v.__class__.__name__.endswith('function_or_method'))): f = self.Function(self, k, parent) f.parentMod = parentMod f.docstring = v.__doc__ diff --git a/pydoctor/sphinx_ext/build_apidocs.py b/pydoctor/sphinx_ext/build_apidocs.py index a5b828885..74b1b501b 100644 --- a/pydoctor/sphinx_ext/build_apidocs.py +++ b/pydoctor/sphinx_ext/build_apidocs.py @@ -49,7 +49,7 @@ def on_build_finished(app: Sphinx, exception: Exception) -> None: runs = app.config.pydoctor_args placeholders = { - 'outdir': app.outdir, + 'outdir': str(app.outdir), } if not isinstance(runs, Mapping): @@ -86,7 +86,7 @@ def on_builder_inited(app: Sphinx) -> None: raise ConfigError("Missing 'pydoctor_args'.") placeholders = { - 'outdir': app.outdir, + 'outdir': str(app.outdir), } runs = config.pydoctor_args diff --git a/pydoctor/templatewriter/__init__.py b/pydoctor/templatewriter/__init__.py index ffaff07a1..7a158a57d 100644 --- a/pydoctor/templatewriter/__init__.py +++ b/pydoctor/templatewriter/__init__.py @@ -1,5 +1,5 @@ """Render pydoctor data as HTML.""" -from typing import Any, Iterable, Iterator, Optional, Union, cast, TYPE_CHECKING +from typing import Any, Iterable, Iterator, Optional, Union, TYPE_CHECKING if TYPE_CHECKING: from typing_extensions import Protocol, runtime_checkable else: @@ -38,8 +38,7 @@ def parse_xml(text: str) -> minidom.Document: Create a L{minidom} representaton of the XML string. """ try: - # TODO: submit a PR to typeshed to add a return type for parseString() - return cast(minidom.Document, minidom.parseString(text)) + return minidom.parseString(text) except Exception as e: raise ValueError(f"Failed to parse template as XML: {e}") from e diff --git a/pydoctor/test/test_sphinx.py b/pydoctor/test/test_sphinx.py index d05274c04..d71e0caf5 100644 --- a/pydoctor/test/test_sphinx.py +++ b/pydoctor/test/test_sphinx.py @@ -110,7 +110,7 @@ def test_generate_empty_functional() -> None: @contextmanager def openFileForWriting(path: str) -> Iterator[io.BytesIO]: yield output - inv_writer._openFileForWriting = openFileForWriting # type: ignore[assignment] + inv_writer._openFileForWriting = openFileForWriting # type: ignore inv_writer.generate(subjects=[], basepath='base-path') diff --git a/readthedocs.yml b/readthedocs.yml index 6802034a9..1a1beb95b 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -4,9 +4,12 @@ version: 2 sphinx: fail_on_warning: false +build: + os: ubuntu-22.04 + tools: + python: "3.11" + python: - version: 3.8 - system_packages: false install: - method: pip path: . diff --git a/setup.cfg b/setup.cfg index 8b10c2cb5..1f00cc431 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,9 +66,9 @@ test = coverage pytest hypothesis - cython-test-exception-raiser==1.0.0 + cython-test-exception-raiser bs4 - Sphinx>=3.5 + Sphinx<7.0.0 pytest-subtests [options.entry_points] From 0eab64f99909b9d770f4bc8af9a6522b973c131f Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Sat, 9 Sep 2023 19:38:11 -0400 Subject: [PATCH 2/5] Use stricter verifications before marking an attribute as constant (#710) * Use stricter verifications before marking an attribute as constant. Do not trigger any warnings when pydoctor cannot make sense of a potential constant attribute. Fix #623 * Say that pydoctor is not a checker. --- README.rst | 6 + pydoctor/astbuilder.py | 187 +++++++++++++++----------- pydoctor/astutils.py | 31 ++++- pydoctor/epydoc/markup/_pyval_repr.py | 35 ++--- pydoctor/test/test_astbuilder.py | 177 +++++++++++++++++++++--- 5 files changed, 312 insertions(+), 124 deletions(-) diff --git a/README.rst b/README.rst index 48a8df2ce..1c71c6ccd 100644 --- a/README.rst +++ b/README.rst @@ -82,6 +82,12 @@ in development scope when possible, when impossible, the theoretical runtime scopes are used. A warning can be reported when an annotation name is ambiguous (can be resolved to different names depending on the scope context) with option ``-v``. +* Use stricter verification before marking an attribute as constant: + - instance variables are never marked as constant + - a variable that has several definitions will not be marked as constant + - a variable declaration under any kind of control flow block will not be marked as constant +* Do not trigger warnings when pydoctor cannot make sense of a potential constant attribute + (pydoctor is not a static checker). * Fix presentation of type aliases in string form. * Improve the AST colorizer to output less parenthesis when it's not required. * Fix colorization of dictionary unpacking. diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index e1fc18004..621cf8cc5 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -16,8 +16,8 @@ from pydoctor import epydoc2stan, model, node2stan, extensions, linker from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval from pydoctor.astutils import (is_none_literal, is_typing_annotation, is_using_annotations, is_using_typing_final, node2dottedname, node2fullname, - is__name__equals__main__, unstring_annotation, iterassign, extract_docstring_linenum, - NodeVisitor) + is__name__equals__main__, unstring_annotation, iterassign, extract_docstring_linenum, get_parents, + NodeVisitor, Parentage) def parseFile(path: Path) -> ast.Module: """Parse the contents of a Python source file.""" @@ -60,18 +60,34 @@ def _handleAliasing( ctx._localNameToFullName_map[target] = full_name return True -def is_constant(obj: model.Attribute) -> bool: + +_CONTROL_FLOW_BLOCKS:Tuple[Type[ast.stmt],...] = (ast.If, ast.While, ast.For, ast.Try, + ast.AsyncFor, ast.With, ast.AsyncWith) +""" +AST types that introduces a new control flow block, potentially conditionnal. +""" +if sys.version_info >= (3, 10): + _CONTROL_FLOW_BLOCKS += (ast.Match,) +if sys.version_info >= (3, 11): + _CONTROL_FLOW_BLOCKS += (ast.TryStar,) + +def is_constant(obj: model.Attribute, + annotation:Optional[ast.expr], + value:Optional[ast.expr]) -> bool: """ Detect if the given assignment is a constant. - To detect whether a assignment is a constant, this checks two things: - - all-caps variable name - - typing.Final annotation + For an assignment to be detected as constant, it should: + - have all-caps variable name or using L{typing.Final} annotation + - not be overriden + - not be defined in a conditionnal block or any other kind of control flow blocks @note: Must be called after setting obj.annotation to detect variables using Final. """ - - return obj.name.isupper() or is_using_typing_final(obj.annotation, obj) + if not is_attribute_overridden(obj, value) and value: + if not any(isinstance(n, _CONTROL_FLOW_BLOCKS) for n in get_parents(value)): + return obj.name.isupper() or is_using_typing_final(annotation, obj) + return False class TypeAliasVisitorExt(extensions.ModuleVisitorExt): """ @@ -168,6 +184,7 @@ def visit_If(self, node: ast.If) -> None: def visit_Module(self, node: ast.Module) -> None: assert self.module.docstring is None + Parentage().visit(node) self.builder.push(self.module, 0) if len(node.body) > 0 and isinstance(node.body[0], ast.Expr) and isinstance(node.body[0].value, ast.Str): @@ -454,39 +471,28 @@ def _handleOldSchoolMethodDecoration(self, target: str, expr: Optional[ast.expr] target_obj.kind = model.DocumentableKind.CLASS_METHOD return True return False - - def _warnsConstantAssigmentOverride(self, obj: model.Attribute, lineno_offset: int) -> None: - obj.report(f'Assignment to constant "{obj.name}" overrides previous assignment ' - f'at line {obj.linenumber}, the original value will not be part of the docs.', - section='ast', lineno_offset=lineno_offset) - - def _warnsConstantReAssigmentInInstance(self, obj: model.Attribute, lineno_offset: int = 0) -> None: - obj.report(f'Assignment to constant "{obj.name}" inside an instance is ignored, this value will not be part of the docs.', - section='ast', lineno_offset=lineno_offset) - - def _handleConstant(self, obj: model.Attribute, value: Optional[ast.expr], lineno: int) -> None: - """Must be called after obj.setLineNumber() to have the right line number in the warning.""" - - if is_attribute_overridden(obj, value): - - if obj.kind in (model.DocumentableKind.CONSTANT, - model.DocumentableKind.VARIABLE, - model.DocumentableKind.CLASS_VARIABLE): - # Module/Class level warning, regular override. - self._warnsConstantAssigmentOverride(obj=obj, lineno_offset=lineno-obj.linenumber) - else: - # Instance level warning caught at the time of the constant detection. - self._warnsConstantReAssigmentInInstance(obj) - - obj.value = value - - obj.kind = model.DocumentableKind.CONSTANT - # A hack to to display variables annotated with Final with the real type instead. - if is_using_typing_final(obj.annotation, obj): - if isinstance(obj.annotation, ast.Subscript): + @classmethod + def _handleConstant(cls, obj:model.Attribute, + annotation:Optional[ast.expr], + value:Optional[ast.expr], + lineno:int, + defaultKind:model.DocumentableKind) -> None: + if is_constant(obj, annotation=annotation, value=value): + obj.kind = model.DocumentableKind.CONSTANT + cls._tweakConstantAnnotation(obj=obj, annotation=annotation, + value=value, lineno=lineno) + elif obj.kind is model.DocumentableKind.CONSTANT: + obj.kind = defaultKind + + @staticmethod + def _tweakConstantAnnotation(obj: model.Attribute, annotation:Optional[ast.expr], + value: Optional[ast.expr], lineno: int) -> None: + # Display variables annotated with Final with the real type instead. + if is_using_typing_final(annotation, obj): + if isinstance(annotation, ast.Subscript): try: - annotation = extract_final_subscript(obj.annotation) + annotation = extract_final_subscript(annotation) except ValueError as e: obj.report(str(e), section='ast', lineno_offset=lineno-obj.linenumber) obj.annotation = _infer_type(value) if value else None @@ -498,11 +504,33 @@ def _handleConstant(self, obj: model.Attribute, value: Optional[ast.expr], linen # Simply ignore it because it's duplication of information. obj.annotation = _infer_type(value) if value else None + @staticmethod + def _storeAttrValue(obj:model.Attribute, new_value:Optional[ast.expr], + augassign:Optional[ast.operator]=None) -> None: + if new_value: + if augassign: + if obj.value: + # We're storing the value of augmented assignemnt value as binop for the sake + # of correctness, but we're not doing anything special with it at the + # moment, nonethless this could be useful for future developments. + # We don't bother reporting warnings, pydoctor is not a checker. + obj.value = ast.BinOp(left=obj.value, op=augassign, right=new_value) + else: + obj.value = new_value + + def _storeCurrentAttr(self, obj:model.Attribute, + augassign:Optional[object]=None) -> None: + if not augassign: + self.builder.currentAttr = obj + else: + self.builder.currentAttr = None + def _handleModuleVar(self, target: str, annotation: Optional[ast.expr], expr: Optional[ast.expr], - lineno: int + lineno: int, + augassign:Optional[ast.operator], ) -> None: if target in MODULE_VARIABLES_META_PARSERS: # This is metadata, not a variable that needs to be documented, @@ -510,9 +538,12 @@ def _handleModuleVar(self, return parent = self.builder.current obj = parent.contents.get(target) - if obj is None: - obj = self.builder.addAttribute(name=target, kind=None, parent=parent) + if augassign: + return + obj = self.builder.addAttribute(name=target, + kind=model.DocumentableKind.VARIABLE, + parent=parent) # If it's not an attribute it means that the name is already denifed as function/class # probably meaning that this attribute is a bound callable. @@ -534,32 +565,29 @@ def _handleModuleVar(self, obj.annotation = annotation obj.setLineNumber(lineno) - if is_constant(obj): - self._handleConstant(obj=obj, value=expr, lineno=lineno) - else: - obj.kind = model.DocumentableKind.VARIABLE - # We store the expr value for all Attribute in order to be able to - # check if they have been initialized or not. - obj.value = expr - - self.builder.currentAttr = obj + self._handleConstant(obj, annotation, expr, lineno, + model.DocumentableKind.VARIABLE) + self._storeAttrValue(obj, expr, augassign) + self._storeCurrentAttr(obj, augassign) def _handleAssignmentInModule(self, target: str, annotation: Optional[ast.expr], expr: Optional[ast.expr], - lineno: int + lineno: int, + augassign:Optional[ast.operator], ) -> None: module = self.builder.current assert isinstance(module, model.Module) if not _handleAliasing(module, target, expr): - self._handleModuleVar(target, annotation, expr, lineno) + self._handleModuleVar(target, annotation, expr, lineno, augassign=augassign) def _handleClassVar(self, name: str, annotation: Optional[ast.expr], expr: Optional[ast.expr], - lineno: int + lineno: int, + augassign:Optional[ast.operator], ) -> None: cls = self.builder.current assert isinstance(cls, model.Class) @@ -570,6 +598,8 @@ def _handleClassVar(self, obj = cast(Optional[model.Attribute], cls.contents.get(name)) if obj is None: + if augassign: + return obj = self.builder.addAttribute(name=name, kind=None, parent=cls) if obj.kind is None: @@ -582,12 +612,10 @@ def _handleClassVar(self, obj.annotation = annotation obj.setLineNumber(lineno) - if is_constant(obj): - self._handleConstant(obj=obj, value=expr, lineno=lineno) - else: - obj.value = expr - - self.builder.currentAttr = obj + self._handleConstant(obj, annotation, expr, lineno, + model.DocumentableKind.CLASS_VARIABLE) + self._storeAttrValue(obj, expr, augassign) + self._storeCurrentAttr(obj, augassign) def _handleInstanceVar(self, name: str, @@ -607,7 +635,6 @@ def _handleInstanceVar(self, # Class variables can only be Attribute, so it's OK to cast because we used _maybeAttribute() above. obj = cast(Optional[model.Attribute], cls.contents.get(name)) if obj is None: - obj = self.builder.addAttribute(name=name, kind=None, parent=cls) if annotation is None and expr is not None: @@ -615,27 +642,22 @@ def _handleInstanceVar(self, obj.annotation = annotation obj.setLineNumber(lineno) - - # Maybe an instance variable overrides a constant, - # so we check before setting the kind to INSTANCE_VARIABLE. - if obj.kind is model.DocumentableKind.CONSTANT: - self._warnsConstantReAssigmentInInstance(obj, lineno_offset=lineno-obj.linenumber) - else: - obj.kind = model.DocumentableKind.INSTANCE_VARIABLE - obj.value = expr - - self.builder.currentAttr = obj + # undonditionnaly set the kind to ivar + obj.kind = model.DocumentableKind.INSTANCE_VARIABLE + self._storeAttrValue(obj, expr) + self._storeCurrentAttr(obj) def _handleAssignmentInClass(self, target: str, annotation: Optional[ast.expr], expr: Optional[ast.expr], - lineno: int + lineno: int, + augassign:Optional[ast.operator], ) -> None: cls = self.builder.current assert isinstance(cls, model.Class) if not _handleAliasing(cls, target, expr): - self._handleClassVar(target, annotation, expr, lineno) + self._handleClassVar(target, annotation, expr, lineno, augassign=augassign) def _handleDocstringUpdate(self, targetNode: ast.expr, @@ -689,17 +711,18 @@ def _handleAssignment(self, targetNode: ast.expr, annotation: Optional[ast.expr], expr: Optional[ast.expr], - lineno: int + lineno: int, + augassign:Optional[ast.operator]=None, ) -> None: if isinstance(targetNode, ast.Name): target = targetNode.id scope = self.builder.current if isinstance(scope, model.Module): - self._handleAssignmentInModule(target, annotation, expr, lineno) + self._handleAssignmentInModule(target, annotation, expr, lineno, augassign=augassign) elif isinstance(scope, model.Class): - if not self._handleOldSchoolMethodDecoration(target, expr): - self._handleAssignmentInClass(target, annotation, expr, lineno) - elif isinstance(targetNode, ast.Attribute): + if augassign or not self._handleOldSchoolMethodDecoration(target, expr): + self._handleAssignmentInClass(target, annotation, expr, lineno, augassign=augassign) + elif isinstance(targetNode, ast.Attribute) and not augassign: value = targetNode.value if targetNode.attr == '__doc__': self._handleDocstringUpdate(value, expr, lineno) @@ -728,6 +751,10 @@ def visit_Assign(self, node: ast.Assign) -> None: def visit_AnnAssign(self, node: ast.AnnAssign) -> None: annotation = unstring_annotation(node.annotation, self.builder.current) self._handleAssignment(node.target, annotation, node.value, node.lineno) + + def visit_AugAssign(self, node:ast.AugAssign) -> None: + self._handleAssignment(node.target, None, node.value, + node.lineno, augassign=node.op) def visit_Expr(self, node: ast.Expr) -> None: value = node.value @@ -904,7 +931,9 @@ def _handlePropertyDef(self, lineno: int ) -> model.Attribute: - attr = self.builder.addAttribute(name=node.name, kind=model.DocumentableKind.PROPERTY, parent=self.builder.current) + attr = self.builder.addAttribute(name=node.name, + kind=model.DocumentableKind.PROPERTY, + parent=self.builder.current) attr.setLineNumber(lineno) if docstring is not None: diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 5677e5c68..8c9154d41 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -6,7 +6,7 @@ import platform import sys from numbers import Number -from typing import Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union +from typing import Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union, cast from inspect import BoundArguments, Signature import ast @@ -402,4 +402,31 @@ def extract_docstring(node: ast.Str) -> Tuple[int, str]: - The docstring to be parsed, cleaned by L{inspect.cleandoc}. """ lineno = extract_docstring_linenum(node) - return lineno, inspect.cleandoc(node.s) \ No newline at end of file + return lineno, inspect.cleandoc(node.s) + +class Parentage(ast.NodeTransformer): + """ + Add C{parent} attribute to ast nodes instances. + """ + # stolen from https://stackoverflow.com/a/68845448 + parent: Optional[ast.AST] = None + + def visit(self, node: ast.AST) -> ast.AST: + setattr(node, 'parent', self.parent) + self.parent = node + node = super().visit(node) + if isinstance(node, ast.AST): + self.parent = getattr(node, 'parent') + return node + +def get_parents(node:ast.AST) -> Iterator[ast.AST]: + """ + Once nodes have the C{.parent} attribute with {Parentage}, use this function + to get a iterator on all parents of the given node up to the root module. + """ + def _yield_parents(n:Optional[ast.AST]) -> Iterator[ast.AST]: + if n: + yield n + p = cast(ast.AST, getattr(n, 'parent', None)) + yield from _yield_parents(p) + yield from _yield_parents(getattr(node, 'parent', None)) \ No newline at end of file diff --git a/pydoctor/epydoc/markup/_pyval_repr.py b/pydoctor/epydoc/markup/_pyval_repr.py index 7eae1918f..b1b8b4985 100644 --- a/pydoctor/epydoc/markup/_pyval_repr.py +++ b/pydoctor/epydoc/markup/_pyval_repr.py @@ -51,7 +51,7 @@ from pydoctor.epydoc.markup import DocstringLinker from pydoctor.epydoc.markup.restructuredtext import ParsedRstDocstring from pydoctor.epydoc.docutils import set_node_attributes, wbr, obj_reference, new_document -from pydoctor.astutils import node2dottedname, bind_args +from pydoctor.astutils import node2dottedname, bind_args, Parentage, get_parents def decode_with_backslashreplace(s: bytes) -> str: r""" @@ -111,21 +111,6 @@ def restore(self, mark: _MarkedColorizerState) -> List[nodes.Node]: del self.result[mark.length:] return trimmed -class _Parentage(ast.NodeTransformer): - """ - Add C{parent} attribute to ast nodes instances. - """ - # stolen from https://stackoverflow.com/a/68845448 - parent: Optional[ast.AST] = None - - def visit(self, node: ast.AST) -> ast.AST: - setattr(node, 'parent', self.parent) - self.parent = node - node = super().visit(node) - if isinstance(node, ast.AST): - self.parent = getattr(node, 'parent') - return node - # TODO: add support for comparators when needed. # _OperatorDelimitier is needed for: # - IfExp @@ -152,10 +137,14 @@ def __init__(self, colorizer: 'PyvalColorizer', state: _ColorizerState, self.marked = state.mark() # We use a hack to populate a "parent" attribute on AST nodes. - # See _Parentage class, applied in PyvalColorizer._colorize_ast() - parent_node: Optional[ast.AST] = getattr(node, 'parent', None) - - if parent_node: + # See astutils.Parentage class, applied in PyvalColorizer._colorize_ast() + try: + parent_node: ast.AST = next(get_parents(node)) + except StopIteration: + return + + # avoid needless parenthesis, since we now collect parents for every nodes + if isinstance(parent_node, (ast.expr, ast.keyword, ast.comprehension)): precedence = astor.op_util.get_op_precedence(node.op) if isinstance(parent_node, (ast.UnaryOp, ast.BinOp, ast.BoolOp)): parent_precedence = astor.op_util.get_op_precedence(parent_node.op) @@ -547,8 +536,10 @@ def _colorize_ast_constant(self, pyval: ast.AST, state: _ColorizerState) -> None def _colorize_ast(self, pyval: ast.AST, state: _ColorizerState) -> None: # Set nodes parent in order to check theirs precedences and add delimiters when needed. - if not getattr(pyval, 'parent', None): - _Parentage().visit(pyval) + try: + next(get_parents(pyval)) + except StopIteration: + Parentage().visit(pyval) if self._is_ast_constant(pyval): self._colorize_ast_constant(pyval, state) diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index a12e74307..a12df406c 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -1829,9 +1829,9 @@ def __init__(**args): assert not captured @systemcls_param -def test_constant_override_in_instace_warns(systemcls: Type[model.System], capsys: CapSys) -> None: +def test_constant_override_in_instace(systemcls: Type[model.System], capsys: CapSys) -> None: """ - It warns when a constant is beeing re defined in instance. But it ignores it's value. + When an instance variable overrides a CONSTANT, it's flagged as INSTANCE_VARIABLE and no warning is raised. """ mod = fromText(''' class Clazz: @@ -1842,18 +1842,13 @@ def __init__(self, **args): ''', systemcls=systemcls, modname="mod") attr = mod.resolveName('Clazz.LANG') assert isinstance(attr, model.Attribute) - assert attr.kind == model.DocumentableKind.CONSTANT - assert attr.value is not None - assert ast.literal_eval(attr.value) == 'EN' - - captured = capsys.readouterr().out - assert "mod:6: Assignment to constant \"LANG\" inside an instance is ignored, this value will not be part of the docs.\n" == captured + assert attr.kind == model.DocumentableKind.INSTANCE_VARIABLE + assert not capsys.readouterr().out @systemcls_param -def test_constant_override_in_instace_warns2(systemcls: Type[model.System], capsys: CapSys) -> None: +def test_constant_override_in_instace_bis(systemcls: Type[model.System], capsys: CapSys) -> None: """ - It warns when a constant is beeing re defined in instance. But it ignores it's value. - Even if the actual constant definition is detected after the instance variable of the same name. + When an instance variable overrides a CONSTANT, it's flagged as INSTANCE_VARIABLE and no warning is raised. """ mod = fromText(''' class Clazz: @@ -1864,15 +1859,13 @@ def __init__(self, **args): ''', systemcls=systemcls, modname="mod") attr = mod.resolveName('Clazz.LANG') assert isinstance(attr, model.Attribute) - assert attr.kind == model.DocumentableKind.CONSTANT + assert attr.kind == model.DocumentableKind.INSTANCE_VARIABLE assert attr.value is not None assert ast.literal_eval(attr.value) == 'EN' - - captured = capsys.readouterr().out - assert "mod:5: Assignment to constant \"LANG\" inside an instance is ignored, this value will not be part of the docs.\n" == captured + assert not capsys.readouterr().out @systemcls_param -def test_constant_override_in_module_warns(systemcls: Type[model.System], capsys: CapSys) -> None: +def test_constant_override_in_module(systemcls: Type[model.System], capsys: CapSys) -> None: mod = fromText(''' """Mod.""" @@ -1883,12 +1876,10 @@ def test_constant_override_in_module_warns(systemcls: Type[model.System], capsys ''', systemcls=systemcls, modname="mod") attr = mod.resolveName('IS_64BITS') assert isinstance(attr, model.Attribute) - assert attr.kind == model.DocumentableKind.CONSTANT + assert attr.kind == model.DocumentableKind.VARIABLE assert attr.value is not None assert ast.literal_eval(attr.value) == True - - captured = capsys.readouterr().out - assert "mod:6: Assignment to constant \"IS_64BITS\" overrides previous assignment at line 4, the original value will not be part of the docs.\n" == captured + assert not capsys.readouterr().out @systemcls_param def test_constant_override_do_not_warns_when_defined_in_class_docstring(systemcls: Type[model.System], capsys: CapSys) -> None: @@ -1927,6 +1918,39 @@ def test_constant_override_do_not_warns_when_defined_in_module_docstring(systemc captured = capsys.readouterr().out assert not captured +@systemcls_param +def test_not_a_constant_module(systemcls: Type[model.System], capsys:CapSys) -> None: + """ + If the constant assignment has any kind of constraint or there are multiple assignments in the scope, + then it's not flagged as a constant. + """ + mod = fromText(''' + while False: + LANG = 'FR' + if True: + THING = 'EN' + OTHER = 1 + OTHER += 1 + E: typing.Final = 2 + E = 4 + LIST = [2.14] + LIST.insert(0,0) + ''', systemcls=systemcls) + assert mod.contents['LANG'].kind is model.DocumentableKind.VARIABLE + assert mod.contents['THING'].kind is model.DocumentableKind.VARIABLE + assert mod.contents['OTHER'].kind is model.DocumentableKind.VARIABLE + assert mod.contents['E'].kind is model.DocumentableKind.VARIABLE + + # all-caps mutables variables are flagged as constant: this is a trade-off + # in between our weeknesses in terms static analysis (that is we don't recognized list modifications) + # and our will to do the right thing and display constant values. + # This issue could be overcome by showing the value of variables with only one assigment no matter + # their kind and restrict the checks to immutable types for a attribute to be flagged as constant. + assert mod.contents['LIST'].kind is model.DocumentableKind.CONSTANT + + # we could warn when a constant is beeing overriden, but we don't: pydoctor is not a checker. + assert not capsys.readouterr().out + @systemcls_param def test__name__equals__main__is_skipped(systemcls: Type[model.System]) -> None: """ @@ -2397,6 +2421,116 @@ def __init__(self): mod = fromText(src, systemcls=systemcls) assert getConstructorsText(mod.contents['Animal']) == "Animal()" +@systemcls_param +def test_augmented_assignment(systemcls: Type[model.System]) -> None: + mod = fromText(''' + var = 1 + var += 3 + ''', systemcls=systemcls) + attr = mod.contents['var'] + assert isinstance(attr, model.Attribute) + assert attr.value + assert astor.to_source(attr.value).strip() == '(1 + 3)' + +@systemcls_param +def test_augmented_assignment_in_class(systemcls: Type[model.System]) -> None: + mod = fromText(''' + class c: + var = 1 + var += 3 + ''', systemcls=systemcls) + attr = mod.contents['c'].contents['var'] + assert isinstance(attr, model.Attribute) + assert attr.value + assert astor.to_source(attr.value).strip() == '(1 + 3)' + + +@systemcls_param +def test_augmented_assignment_conditionnal_else_ignored(systemcls: Type[model.System]) -> None: + """ + The If.body branch is the only one in use. + """ + mod = fromText(''' + var = 1 + if something(): + var += 3 + else: + var += 4 + ''', systemcls=systemcls) + attr = mod.contents['var'] + assert isinstance(attr, model.Attribute) + assert attr.value + assert astor.to_source(attr.value).strip() == '(1 + 3)' + +@systemcls_param +def test_augmented_assignment_conditionnal_multiple_assignments(systemcls: Type[model.System]) -> None: + """ + The If.body branch is the only one in use, but several Ifs which have + theoritical exclusive conditions might be wrongly interpreted. + """ + mod = fromText(''' + var = 1 + if something(): + var += 3 + if not_something(): + var += 4 + ''', systemcls=systemcls) + attr = mod.contents['var'] + assert isinstance(attr, model.Attribute) + assert attr.value + assert astor.to_source(attr.value).strip() == '(1 + 3 + 4)' + +@systemcls_param +def test_augmented_assignment_instance_var(systemcls: Type[model.System]) -> None: + """ + Augmented assignments in instance var are not analyzed. + """ + mod = fromText(''' + class c: + def __init__(self, var): + self.var = 1 + self.var += var + ''') + attr = mod.contents['c'].contents['var'] + assert isinstance(attr, model.Attribute) + assert attr.value + assert astor.to_source(attr.value).strip() == '(1)' + +@systemcls_param +def test_augmented_assignment_not_suitable_for_inline_docstring(systemcls: Type[model.System]) -> None: + """ + Augmented assignments cannot have docstring attached. + """ + mod = fromText(''' + var = 1 + var += 1 + """ + this is not a docstring + """ + class c: + var = 1 + var += 1 + """ + this is not a docstring + """ + ''') + attr = mod.contents['var'] + assert not attr.docstring + attr = mod.contents['c'].contents['var'] + assert not attr.docstring + +@systemcls_param +def test_augmented_assignment_alone_is_not_documented(systemcls: Type[model.System]) -> None: + mod = fromText(''' + var += 1 + class c: + var += 1 + ''') + + assert 'var' not in mod.contents + assert 'var' not in mod.contents['c'].contents + + @systemcls_param def test_typealias_unstring(systemcls: Type[model.System]) -> None: """ @@ -2413,4 +2547,5 @@ def test_typealias_unstring(systemcls: Type[model.System]) -> None: assert typealias.value with pytest.raises(StopIteration): # there is not Constant nodes in the type alias anymore - next(n for n in ast.walk(typealias.value) if isinstance(n, ast.Constant)) \ No newline at end of file + next(n for n in ast.walk(typealias.value) if isinstance(n, ast.Constant)) + From f8e80c9fd6bd6acc5f481494200052d4ea464608 Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Sat, 9 Sep 2023 23:07:15 -0400 Subject: [PATCH 3/5] Explicit annotation wins over inferred type (#691) * Fix #690 --- README.rst | 1 + pydoctor/astbuilder.py | 100 ++++++++++-------------------- pydoctor/astutils.py | 56 ++++++++++++++++- pydoctor/extensions/attrs.py | 2 +- pydoctor/model.py | 12 ++-- pydoctor/test/test_astbuilder.py | 103 ++++++++++++++++++++++++++++++- 6 files changed, 195 insertions(+), 79 deletions(-) diff --git a/README.rst b/README.rst index 1c71c6ccd..feb4e91ac 100644 --- a/README.rst +++ b/README.rst @@ -82,6 +82,7 @@ in development scope when possible, when impossible, the theoretical runtime scopes are used. A warning can be reported when an annotation name is ambiguous (can be resolved to different names depending on the scope context) with option ``-v``. +* Ensure that explicit annotation are honored when there are multiple declarations of the same name. * Use stricter verification before marking an attribute as constant: - instance variables are never marked as constant - a variable that has several definitions will not be marked as constant diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index 621cf8cc5..3622cb4f8 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -16,9 +16,10 @@ from pydoctor import epydoc2stan, model, node2stan, extensions, linker from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval from pydoctor.astutils import (is_none_literal, is_typing_annotation, is_using_annotations, is_using_typing_final, node2dottedname, node2fullname, - is__name__equals__main__, unstring_annotation, iterassign, extract_docstring_linenum, get_parents, + is__name__equals__main__, unstring_annotation, iterassign, extract_docstring_linenum, infer_type, get_parents, NodeVisitor, Parentage) + def parseFile(path: Path) -> ast.Module: """Parse the contents of a Python source file.""" with open(path, 'rb') as f: @@ -173,6 +174,17 @@ def __init__(self, builder: 'ASTBuilder', module: model.Module): self.system = builder.system self.module = module + def _infer_attr_annotations(self, scope: model.Documentable) -> None: + # Infer annotation when leaving scope so explicit + # annotations take precedence. + for attrib in scope.contents.values(): + if not isinstance(attrib, model.Attribute): + continue + # If this attribute has not explicit annotation, + # infer its type from it's ast expression. + if attrib.annotation is None and attrib.value is not None: + # do not override explicit annotation + attrib.annotation = infer_type(attrib.value) def visit_If(self, node: ast.If) -> None: if isinstance(node.test, ast.Compare): @@ -192,6 +204,7 @@ def visit_Module(self, node: ast.Module) -> None: epydoc2stan.extract_fields(self.module) def depart_Module(self, node: ast.Module) -> None: + self._infer_attr_annotations(self.builder.current) self.builder.pop(self.module) def visit_ClassDef(self, node: ast.ClassDef) -> None: @@ -275,6 +288,7 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None: def depart_ClassDef(self, node: ast.ClassDef) -> None: + self._infer_attr_annotations(self.builder.current) self.builder.popClass() @@ -495,14 +509,22 @@ def _tweakConstantAnnotation(obj: model.Attribute, annotation:Optional[ast.expr] annotation = extract_final_subscript(annotation) except ValueError as e: obj.report(str(e), section='ast', lineno_offset=lineno-obj.linenumber) - obj.annotation = _infer_type(value) if value else None + obj.annotation = infer_type(value) if value else None else: # Will not display as "Final[str]" but rather only "str" obj.annotation = annotation else: # Just plain "Final" annotation. # Simply ignore it because it's duplication of information. - obj.annotation = _infer_type(value) if value else None + obj.annotation = infer_type(value) if value else None + + @staticmethod + def _setAttributeAnnotation(obj: model.Attribute, + annotation: Optional[ast.expr],) -> None: + if annotation is not None: + # TODO: What to do when an attribute has several explicit annotations? + # (mypy reports a warning in these kind of cases) + obj.annotation = annotation @staticmethod def _storeAttrValue(obj:model.Attribute, new_value:Optional[ast.expr], @@ -558,11 +580,9 @@ def _handleModuleVar(self, if not isinstance(obj, model.Attribute): return - - if annotation is None and expr is not None: - annotation = _infer_type(expr) - obj.annotation = annotation + self._setAttributeAnnotation(obj, annotation) + obj.setLineNumber(lineno) self._handleConstant(obj, annotation, expr, lineno, @@ -605,11 +625,8 @@ def _handleClassVar(self, if obj.kind is None: obj.kind = model.DocumentableKind.CLASS_VARIABLE - if expr is not None: - if annotation is None: - annotation = _infer_type(expr) - - obj.annotation = annotation + self._setAttributeAnnotation(obj, annotation) + obj.setLineNumber(lineno) self._handleConstant(obj, annotation, expr, lineno, @@ -637,10 +654,8 @@ def _handleInstanceVar(self, if obj is None: obj = self.builder.addAttribute(name=name, kind=None, parent=cls) - if annotation is None and expr is not None: - annotation = _infer_type(expr) - - obj.annotation = annotation + self._setAttributeAnnotation(obj, annotation) + obj.setLineNumber(lineno) # undonditionnaly set the kind to ivar obj.kind = model.DocumentableKind.INSTANCE_VARIABLE @@ -1043,59 +1058,6 @@ def __repr__(self) -> str: """ return '%s' % super().__repr__() - -def _infer_type(expr: ast.expr) -> Optional[ast.expr]: - """Infer an expression's type. - @param expr: The expression's AST. - @return: A type annotation, or None if the expression has no obvious type. - """ - try: - value: object = ast.literal_eval(expr) - except (ValueError, TypeError): - return None - else: - ann = _annotation_for_value(value) - if ann is None: - return None - else: - return ast.fix_missing_locations(ast.copy_location(ann, expr)) - -def _annotation_for_value(value: object) -> Optional[ast.expr]: - if value is None: - return None - name = type(value).__name__ - if isinstance(value, (dict, list, set, tuple)): - ann_elem = _annotation_for_elements(value) - if isinstance(value, dict): - ann_value = _annotation_for_elements(value.values()) - if ann_value is None: - ann_elem = None - elif ann_elem is not None: - ann_elem = ast.Tuple(elts=[ann_elem, ann_value]) - if ann_elem is not None: - if name == 'tuple': - ann_elem = ast.Tuple(elts=[ann_elem, ast.Ellipsis()]) - return ast.Subscript(value=ast.Name(id=name), - slice=ast.Index(value=ann_elem)) - return ast.Name(id=name) - -def _annotation_for_elements(sequence: Iterable[object]) -> Optional[ast.expr]: - names = set() - for elem in sequence: - ann = _annotation_for_value(elem) - if isinstance(ann, ast.Name): - names.add(ann.id) - else: - # Nested sequences are too complex. - return None - if len(names) == 1: - name = names.pop() - return ast.Name(id=name) - else: - # Empty sequence or no uniform type. - return None - - DocumentableT = TypeVar('DocumentableT', bound=model.Documentable) class ASTBuilder: diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 8c9154d41..36fa38293 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -404,6 +404,59 @@ def extract_docstring(node: ast.Str) -> Tuple[int, str]: lineno = extract_docstring_linenum(node) return lineno, inspect.cleandoc(node.s) + +def infer_type(expr: ast.expr) -> Optional[ast.expr]: + """Infer a literal expression's type. + @param expr: The expression's AST. + @return: A type annotation, or None if the expression has no obvious type. + """ + try: + value: object = ast.literal_eval(expr) + except (ValueError, TypeError): + return None + else: + ann = _annotation_for_value(value) + if ann is None: + return None + else: + return ast.fix_missing_locations(ast.copy_location(ann, expr)) + +def _annotation_for_value(value: object) -> Optional[ast.expr]: + if value is None: + return None + name = type(value).__name__ + if isinstance(value, (dict, list, set, tuple)): + ann_elem = _annotation_for_elements(value) + if isinstance(value, dict): + ann_value = _annotation_for_elements(value.values()) + if ann_value is None: + ann_elem = None + elif ann_elem is not None: + ann_elem = ast.Tuple(elts=[ann_elem, ann_value]) + if ann_elem is not None: + if name == 'tuple': + ann_elem = ast.Tuple(elts=[ann_elem, ast.Ellipsis()]) + return ast.Subscript(value=ast.Name(id=name), + slice=ast.Index(value=ann_elem)) + return ast.Name(id=name) + +def _annotation_for_elements(sequence: Iterable[object]) -> Optional[ast.expr]: + names = set() + for elem in sequence: + ann = _annotation_for_value(elem) + if isinstance(ann, ast.Name): + names.add(ann.id) + else: + # Nested sequences are too complex. + return None + if len(names) == 1: + name = names.pop() + return ast.Name(id=name) + else: + # Empty sequence or no uniform type. + return None + + class Parentage(ast.NodeTransformer): """ Add C{parent} attribute to ast nodes instances. @@ -429,4 +482,5 @@ def _yield_parents(n:Optional[ast.AST]) -> Iterator[ast.AST]: yield n p = cast(ast.AST, getattr(n, 'parent', None)) yield from _yield_parents(p) - yield from _yield_parents(getattr(node, 'parent', None)) \ No newline at end of file + yield from _yield_parents(getattr(node, 'parent', None)) + diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 50dccc7b6..364b41e22 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -108,7 +108,7 @@ def annotation_from_attrib( return astutils.unstring_annotation(typ, ctx) default = args.arguments.get('default') if default is not None: - return astbuilder._infer_type(default) + return astutils.infer_type(default) return None class ModuleVisitor(extensions.ModuleVisitorExt): diff --git a/pydoctor/model.py b/pydoctor/model.py index 310529fe0..7a720f5e1 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -840,7 +840,7 @@ class FunctionOverload: class Attribute(Inheritable): kind: Optional[DocumentableKind] = DocumentableKind.ATTRIBUTE - annotation: Optional[ast.expr] + annotation: Optional[ast.expr] = None decorators: Optional[Sequence[ast.expr]] = None value: Optional[ast.expr] = None """ @@ -1436,21 +1436,19 @@ def postProcess(self) -> None: without the risk of drawing incorrect conclusions because modules were not fully processed yet. """ - - # default post-processing includes: - # - Processing of subclasses - # - MRO computing. - # - Lookup of constructors - # - Checking whether the class is an exception for cls in self.objectsOfType(Class): + # Initiate the MROs cls._init_mro() + # Lookup of constructors cls._init_constructors() + # Compute subclasses for b in cls.baseobjects: if b is not None: b.subclasses.append(cls) + # Checking whether the class is an exception if is_exception(cls): cls.kind = DocumentableKind.EXCEPTION diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index a12df406c..8edba83aa 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -2421,6 +2421,108 @@ def __init__(self): mod = fromText(src, systemcls=systemcls) assert getConstructorsText(mod.contents['Animal']) == "Animal()" +@systemcls_param +def test_explicit_annotation_wins_over_inferred_type(systemcls: Type[model.System]) -> None: + """ + Explicit annotations are the preffered way of presenting the type of an attribute. + """ + src = '''\ + class Stuff(object): + thing: List[Tuple[Thing, ...]] + def __init__(self): + self.thing = [] + ''' + mod = fromText(src, systemcls=systemcls, modname='mod') + thing = mod.system.allobjects['mod.Stuff.thing'] + assert flatten_text(epydoc2stan.type2stan(thing)) == "List[Tuple[Thing, ...]]" #type:ignore + + src = '''\ + class Stuff(object): + thing = [] + def __init__(self): + self.thing: List[Tuple[Thing, ...]] = [] + ''' + mod = fromText(src, systemcls=systemcls, modname='mod') + thing = mod.system.allobjects['mod.Stuff.thing'] + assert flatten_text(epydoc2stan.type2stan(thing)) == "List[Tuple[Thing, ...]]" #type:ignore + +@systemcls_param +def test_explicit_inherited_annotation_looses_over_inferred_type(systemcls: Type[model.System]) -> None: + """ + Annotation are of inherited. + """ + src = '''\ + class _Stuff(object): + thing: List[Tuple[Thing, ...]] + class Stuff(_Stuff): + def __init__(self): + self.thing = [] + ''' + mod = fromText(src, systemcls=systemcls, modname='mod') + thing = mod.system.allobjects['mod.Stuff.thing'] + assert flatten_text(epydoc2stan.type2stan(thing)) == "list" #type:ignore + +@systemcls_param +def test_inferred_type_override(systemcls: Type[model.System]) -> None: + """ + The last visited value will be used to infer the type annotation + of an unnanotated attribute. + """ + src = '''\ + class Stuff(object): + thing = 1 + def __init__(self): + self.thing = (1,2) + ''' + mod = fromText(src, systemcls=systemcls, modname='mod') + thing = mod.system.allobjects['mod.Stuff.thing'] + assert flatten_text(epydoc2stan.type2stan(thing)) == "tuple[int, ...]" #type:ignore + +@systemcls_param +def test_inferred_type_is_not_propagated_to_subclasses(systemcls: Type[model.System]) -> None: + """ + Inferred type annotation should not be propagated to subclasses. + """ + src = '''\ + class _Stuff(object): + def __init__(self): + self.thing = [] + class Stuff(_Stuff): + def __init__(self, thing): + self.thing = thing + ''' + mod = fromText(src, systemcls=systemcls, modname='mod') + thing = mod.system.allobjects['mod.Stuff.thing'] + assert epydoc2stan.type2stan(thing) is None + + +@systemcls_param +def test_inherited_type_is_not_propagated_to_subclasses(systemcls: Type[model.System]) -> None: + """ + We can't repliably propage the annotations from one class to it's subclass because of + issue https://github.com/twisted/pydoctor/issues/295. + """ + src1 = '''\ + class _s:... + class _Stuff(object): + def __init__(self): + self.thing:_s = [] + ''' + src2 = '''\ + from base import _Stuff, _s + class Stuff(_Stuff): + def __init__(self, thing): + self.thing = thing + __all__=['Stuff', '_s'] + ''' + system = systemcls() + builder = system.systemBuilder(system) + builder.addModuleString(src1, 'base') + builder.addModuleString(src2, 'mod') + builder.buildModules() + thing = system.allobjects['mod.Stuff.thing'] + assert epydoc2stan.type2stan(thing) is None + @systemcls_param def test_augmented_assignment(systemcls: Type[model.System]) -> None: mod = fromText(''' @@ -2530,7 +2632,6 @@ class c: assert 'var' not in mod.contents assert 'var' not in mod.contents['c'].contents - @systemcls_param def test_typealias_unstring(systemcls: Type[model.System]) -> None: """ From 4e05370f58aa33170bd0aa275e623fd8bf2925f0 Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Sat, 9 Sep 2023 23:18:27 -0400 Subject: [PATCH 4/5] A class variable that overrides an instance variable should still display as instance variable (#692) * Fix #671 --- README.rst | 1 + pydoctor/model.py | 17 ++++++++ pydoctor/test/test_astbuilder.py | 70 ++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+) diff --git a/README.rst b/README.rst index feb4e91ac..8a9c2b594 100644 --- a/README.rst +++ b/README.rst @@ -96,6 +96,7 @@ in development * Add highlighting when clicking on "View In Hierarchy" link from class page. * Recognize variadic generics type variables (PEP 646). * Fix support for introspection of cython3 generated modules. +* Instance variables are marked as such across subclasses. pydoctor 23.4.1 ^^^^^^^^^^^^^^^ diff --git a/pydoctor/model.py b/pydoctor/model.py index 7a720f5e1..2cb8cc375 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -1452,6 +1452,9 @@ def postProcess(self) -> None: if is_exception(cls): cls.kind = DocumentableKind.EXCEPTION + for attrib in self.objectsOfType(Attribute): + _inherits_instance_variable_kind(attrib) + for post_processor in self._post_processors: post_processor(self) @@ -1463,6 +1466,20 @@ def fetchIntersphinxInventories(self, cache: CacheT) -> None: for url in self.options.intersphinx: self.intersphinx.update(cache, url) +def _inherits_instance_variable_kind(attr: Attribute) -> None: + """ + If any of the inherited members of a class variable is an instance variable, + then the subclass' class variable become an instance variable as well. + """ + if attr.kind is not DocumentableKind.CLASS_VARIABLE: + return + docsources = attr.docsources() + next(docsources) + for inherited in docsources: + if inherited.kind is DocumentableKind.INSTANCE_VARIABLE: + attr.kind = DocumentableKind.INSTANCE_VARIABLE + break + def get_docstring( obj: Documentable ) -> Tuple[Optional[str], Optional[Documentable]]: diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index 8edba83aa..b5af2de2e 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -2421,6 +2421,76 @@ def __init__(self): mod = fromText(src, systemcls=systemcls) assert getConstructorsText(mod.contents['Animal']) == "Animal()" +@systemcls_param +def test_class_var_override(systemcls: Type[model.System]) -> None: + + src = '''\ + from number import Number + class Thing(object): + def __init__(self): + self.var: Number = 1 + class Stuff(Thing): + var:float + ''' + + mod = fromText(src, systemcls=systemcls, modname='mod') + var = mod.system.allobjects['mod.Stuff.var'] + assert var.kind == model.DocumentableKind.INSTANCE_VARIABLE + +@systemcls_param +def test_class_var_override_traverse_subclasses(systemcls: Type[model.System]) -> None: + + src = '''\ + from number import Number + class Thing(object): + def __init__(self): + self.var: Number = 1 + class _Stuff(Thing): + ... + class Stuff(_Stuff): + var:float + ''' + + mod = fromText(src, systemcls=systemcls, modname='mod') + var = mod.system.allobjects['mod.Stuff.var'] + assert var.kind == model.DocumentableKind.INSTANCE_VARIABLE + + src = '''\ + from number import Number + class Thing(object): + def __init__(self): + self.var: Optional[Number] = 0 + class _Stuff(Thing): + var = None + class Stuff(_Stuff): + var: float + ''' + + mod = fromText(src, systemcls=systemcls, modname='mod') + var = mod.system.allobjects['mod._Stuff.var'] + assert var.kind == model.DocumentableKind.INSTANCE_VARIABLE + + mod = fromText(src, systemcls=systemcls, modname='mod') + var = mod.system.allobjects['mod.Stuff.var'] + assert var.kind == model.DocumentableKind.INSTANCE_VARIABLE + +def test_class_var_override_attrs() -> None: + + systemcls = AttrsSystem + + src = '''\ + import attr + @attr.s + class Thing(object): + var = attr.ib() + class Stuff(Thing): + var: float + ''' + + mod = fromText(src, systemcls=systemcls, modname='mod') + var = mod.system.allobjects['mod.Stuff.var'] + assert var.kind == model.DocumentableKind.INSTANCE_VARIABLE + @systemcls_param def test_explicit_annotation_wins_over_inferred_type(systemcls: Type[model.System]) -> None: """ From f03223c4ca80c411b80fad2f2549cf351e6d410d Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Sun, 24 Sep 2023 11:29:11 -0400 Subject: [PATCH 5/5] Release pydoctor 23.9.0 (#734) * Bump to version 23.9.0 --- .github/workflows/system.yaml | 9 +-------- README.rst | 5 +++++ setup.cfg | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/system.yaml b/.github/workflows/system.yaml index a0b15b8dc..b7d91f217 100644 --- a/.github/workflows/system.yaml +++ b/.github/workflows/system.yaml @@ -18,16 +18,9 @@ jobs: - uses: actions/checkout@v2 - name: Set up CPython - if: ${{ matrix.tox_target == 'python-igraph-apidocs' }} uses: actions/setup-python@v2 with: - python-version: '3.8' - - - name: Set up PyPy - if: ${{ matrix.tox_target != 'python-igraph-apidocs' }} - uses: actions/setup-python@v2 - with: - python-version: 'pypy-3.6' + python-version: '3.11' - name: Install tox run: | diff --git a/README.rst b/README.rst index 8a9c2b594..b1cfe2376 100644 --- a/README.rst +++ b/README.rst @@ -76,6 +76,11 @@ What's New? in development ^^^^^^^^^^^^^^ +pydoctor 23.9.0 +^^^^^^^^^^^^^^^ + +This is the last major release to support Python 3.6. + * Do not show `**kwargs` when keywords are specifically documented with the `keyword` field and no specific documentation is given for the `**kwargs` entry. * Fix annotation resolution edge cases: names are resolved in the context of the module diff --git a/setup.cfg b/setup.cfg index 1f00cc431..71ae2f469 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pydoctor -version = 23.4.1.dev0 +version = 23.9.0.dev0 author = Michael Hudson-Doyle author_email = micahel@gmail.com maintainer = Maarten ter Huurne