diff --git a/README.rst b/README.rst index 48a8df2ce..8a9c2b594 100644 --- a/README.rst +++ b/README.rst @@ -82,6 +82,13 @@ in development scope when possible, when impossible, the theoretical runtime scopes are used. A warning can be reported when an annotation name is ambiguous (can be resolved to different names depending on the scope context) with option ``-v``. +* Ensure that explicit annotation are honored when there are multiple declarations of the same name. +* Use stricter verification before marking an attribute as constant: + - instance variables are never marked as constant + - a variable that has several definitions will not be marked as constant + - a variable declaration under any kind of control flow block will not be marked as constant +* Do not trigger warnings when pydoctor cannot make sense of a potential constant attribute + (pydoctor is not a static checker). * Fix presentation of type aliases in string form. * Improve the AST colorizer to output less parenthesis when it's not required. * Fix colorization of dictionary unpacking. @@ -89,6 +96,7 @@ in development * Add highlighting when clicking on "View In Hierarchy" link from class page. * Recognize variadic generics type variables (PEP 646). * Fix support for introspection of cython3 generated modules. +* Instance variables are marked as such across subclasses. pydoctor 23.4.1 ^^^^^^^^^^^^^^^ diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index 7708f9287..4ace10bc4 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -16,8 +16,9 @@ from pydoctor import epydoc2stan, model, node2stan, extensions, linker from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval from pydoctor.astutils import (is_none_literal, is_typing_annotation, is_using_annotations, is_using_typing_final, node2dottedname, node2fullname, - is__name__equals__main__, unstring_annotation, iterassign, extract_docstring_linenum, - NodeVisitor) + is__name__equals__main__, unstring_annotation, iterassign, extract_docstring_linenum, infer_type, get_parents, + NodeVisitor, Parentage) + def parseFile(path: Path) -> ast.Module: """Parse the contents of a Python source file.""" @@ -60,18 +61,34 @@ def _handleAliasing( ctx._localNameToFullName_map[target] = full_name return True -def is_constant(obj: model.Attribute) -> bool: + +_CONTROL_FLOW_BLOCKS:Tuple[Type[ast.stmt],...] = (ast.If, ast.While, ast.For, ast.Try, + ast.AsyncFor, ast.With, ast.AsyncWith) +""" +AST types that introduces a new control flow block, potentially conditionnal. +""" +if sys.version_info >= (3, 10): + _CONTROL_FLOW_BLOCKS += (ast.Match,) +if sys.version_info >= (3, 11): + _CONTROL_FLOW_BLOCKS += (ast.TryStar,) + +def is_constant(obj: model.Attribute, + annotation:Optional[ast.expr], + value:Optional[ast.expr]) -> bool: """ Detect if the given assignment is a constant. - To detect whether a assignment is a constant, this checks two things: - - all-caps variable name - - typing.Final annotation + For an assignment to be detected as constant, it should: + - have all-caps variable name or using L{typing.Final} annotation + - not be overriden + - not be defined in a conditionnal block or any other kind of control flow blocks @note: Must be called after setting obj.annotation to detect variables using Final. """ - - return obj.name.isupper() or is_using_typing_final(obj.annotation, obj) + if not is_attribute_overridden(obj, value) and value: + if not any(isinstance(n, _CONTROL_FLOW_BLOCKS) for n in get_parents(value)): + return obj.name.isupper() or is_using_typing_final(annotation, obj) + return False class TypeAliasVisitorExt(extensions.ModuleVisitorExt): """ @@ -157,6 +174,17 @@ def __init__(self, builder: 'ASTBuilder', module: model.Module): self.system = builder.system self.module = module + def _infer_attr_annotations(self, scope: model.Documentable) -> None: + # Infer annotation when leaving scope so explicit + # annotations take precedence. + for attrib in scope.contents.values(): + if not isinstance(attrib, model.Attribute): + continue + # If this attribute has not explicit annotation, + # infer its type from it's ast expression. + if attrib.annotation is None and attrib.value is not None: + # do not override explicit annotation + attrib.annotation = infer_type(attrib.value) def visit_If(self, node: ast.If) -> None: if isinstance(node.test, ast.Compare): @@ -168,6 +196,7 @@ def visit_If(self, node: ast.If) -> None: def visit_Module(self, node: ast.Module) -> None: assert self.module.docstring is None + Parentage().visit(node) self.builder.push(self.module, 0) if len(node.body) > 0 and isinstance(node.body[0], ast.Expr) and isinstance(node.body[0].value, ast.Str): @@ -175,6 +204,7 @@ def visit_Module(self, node: ast.Module) -> None: epydoc2stan.extract_fields(self.module) def depart_Module(self, node: ast.Module) -> None: + self._infer_attr_annotations(self.builder.current) self.builder.pop(self.module) def visit_ClassDef(self, node: ast.ClassDef) -> None: @@ -258,6 +288,7 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None: def depart_ClassDef(self, node: ast.ClassDef) -> None: + self._infer_attr_annotations(self.builder.current) self.builder.popClass() @@ -454,55 +485,74 @@ def _handleOldSchoolMethodDecoration(self, target: str, expr: Optional[ast.expr] target_obj.kind = model.DocumentableKind.CLASS_METHOD return True return False - - def _warnsConstantAssigmentOverride(self, obj: model.Attribute, lineno_offset: int) -> None: - obj.report(f'Assignment to constant "{obj.name}" overrides previous assignment ' - f'at line {obj.linenumber}, the original value will not be part of the docs.', - section='ast', lineno_offset=lineno_offset) - - def _warnsConstantReAssigmentInInstance(self, obj: model.Attribute, lineno_offset: int = 0) -> None: - obj.report(f'Assignment to constant "{obj.name}" inside an instance is ignored, this value will not be part of the docs.', - section='ast', lineno_offset=lineno_offset) - - def _handleConstant(self, obj: model.Attribute, value: Optional[ast.expr], lineno: int) -> None: - """Must be called after obj.setLineNumber() to have the right line number in the warning.""" - - if is_attribute_overridden(obj, value): - - if obj.kind in (model.DocumentableKind.CONSTANT, - model.DocumentableKind.VARIABLE, - model.DocumentableKind.CLASS_VARIABLE): - # Module/Class level warning, regular override. - self._warnsConstantAssigmentOverride(obj=obj, lineno_offset=lineno-obj.linenumber) - else: - # Instance level warning caught at the time of the constant detection. - self._warnsConstantReAssigmentInInstance(obj) - obj.value = value - - obj.kind = model.DocumentableKind.CONSTANT - - # A hack to to display variables annotated with Final with the real type instead. - if is_using_typing_final(obj.annotation, obj): - if isinstance(obj.annotation, ast.Subscript): + @classmethod + def _handleConstant(cls, obj:model.Attribute, + annotation:Optional[ast.expr], + value:Optional[ast.expr], + lineno:int, + defaultKind:model.DocumentableKind) -> None: + if is_constant(obj, annotation=annotation, value=value): + obj.kind = model.DocumentableKind.CONSTANT + cls._tweakConstantAnnotation(obj=obj, annotation=annotation, + value=value, lineno=lineno) + elif obj.kind is model.DocumentableKind.CONSTANT: + obj.kind = defaultKind + + @staticmethod + def _tweakConstantAnnotation(obj: model.Attribute, annotation:Optional[ast.expr], + value: Optional[ast.expr], lineno: int) -> None: + # Display variables annotated with Final with the real type instead. + if is_using_typing_final(annotation, obj): + if isinstance(annotation, ast.Subscript): try: - annotation = extract_final_subscript(obj.annotation) + annotation = extract_final_subscript(annotation) except ValueError as e: obj.report(str(e), section='ast', lineno_offset=lineno-obj.linenumber) - obj.annotation = _infer_type(value) if value else None + obj.annotation = infer_type(value) if value else None else: # Will not display as "Final[str]" but rather only "str" obj.annotation = annotation else: # Just plain "Final" annotation. # Simply ignore it because it's duplication of information. - obj.annotation = _infer_type(value) if value else None + obj.annotation = infer_type(value) if value else None + + @staticmethod + def _setAttributeAnnotation(obj: model.Attribute, + annotation: Optional[ast.expr],) -> None: + if annotation is not None: + # TODO: What to do when an attribute has several explicit annotations? + # (mypy reports a warning in these kind of cases) + obj.annotation = annotation + + @staticmethod + def _storeAttrValue(obj:model.Attribute, new_value:Optional[ast.expr], + augassign:Optional[ast.operator]=None) -> None: + if new_value: + if augassign: + if obj.value: + # We're storing the value of augmented assignemnt value as binop for the sake + # of correctness, but we're not doing anything special with it at the + # moment, nonethless this could be useful for future developments. + # We don't bother reporting warnings, pydoctor is not a checker. + obj.value = ast.BinOp(left=obj.value, op=augassign, right=new_value) + else: + obj.value = new_value + + def _storeCurrentAttr(self, obj:model.Attribute, + augassign:Optional[object]=None) -> None: + if not augassign: + self.builder.currentAttr = obj + else: + self.builder.currentAttr = None def _handleModuleVar(self, target: str, annotation: Optional[ast.expr], expr: Optional[ast.expr], - lineno: int + lineno: int, + augassign:Optional[ast.operator], ) -> None: if target in MODULE_VARIABLES_META_PARSERS: # This is metadata, not a variable that needs to be documented, @@ -510,9 +560,12 @@ def _handleModuleVar(self, return parent = self.builder.current obj = parent.contents.get(target) - if obj is None: - obj = self.builder.addAttribute(name=target, kind=None, parent=parent) + if augassign: + return + obj = self.builder.addAttribute(name=target, + kind=model.DocumentableKind.VARIABLE, + parent=parent) # If it's not an attribute it means that the name is already denifed as function/class # probably meaning that this attribute is a bound callable. @@ -527,39 +580,34 @@ def _handleModuleVar(self, if not isinstance(obj, model.Attribute): return - - if annotation is None and expr is not None: - annotation = _infer_type(expr) - obj.annotation = annotation + self._setAttributeAnnotation(obj, annotation) + obj.setLineNumber(lineno) - if is_constant(obj): - self._handleConstant(obj=obj, value=expr, lineno=lineno) - else: - obj.kind = model.DocumentableKind.VARIABLE - # We store the expr value for all Attribute in order to be able to - # check if they have been initialized or not. - obj.value = expr - - self.builder.currentAttr = obj + self._handleConstant(obj, annotation, expr, lineno, + model.DocumentableKind.VARIABLE) + self._storeAttrValue(obj, expr, augassign) + self._storeCurrentAttr(obj, augassign) def _handleAssignmentInModule(self, target: str, annotation: Optional[ast.expr], expr: Optional[ast.expr], - lineno: int + lineno: int, + augassign:Optional[ast.operator], ) -> None: module = self.builder.current assert isinstance(module, model.Module) if not _handleAliasing(module, target, expr): - self._handleModuleVar(target, annotation, expr, lineno) + self._handleModuleVar(target, annotation, expr, lineno, augassign=augassign) def _handleClassVar(self, name: str, annotation: Optional[ast.expr], expr: Optional[ast.expr], - lineno: int + lineno: int, + augassign:Optional[ast.operator], ) -> None: cls = self.builder.current assert isinstance(cls, model.Class) @@ -570,24 +618,21 @@ def _handleClassVar(self, obj = cast(Optional[model.Attribute], cls.contents.get(name)) if obj is None: + if augassign: + return obj = self.builder.addAttribute(name=name, kind=None, parent=cls) if obj.kind is None: obj.kind = model.DocumentableKind.CLASS_VARIABLE - if expr is not None: - if annotation is None: - annotation = _infer_type(expr) - - obj.annotation = annotation - obj.setLineNumber(lineno) + self._setAttributeAnnotation(obj, annotation) - if is_constant(obj): - self._handleConstant(obj=obj, value=expr, lineno=lineno) - else: - obj.value = expr + obj.setLineNumber(lineno) - self.builder.currentAttr = obj + self._handleConstant(obj, annotation, expr, lineno, + model.DocumentableKind.CLASS_VARIABLE) + self._storeAttrValue(obj, expr, augassign) + self._storeCurrentAttr(obj, augassign) def _handleInstanceVar(self, name: str, @@ -607,35 +652,27 @@ def _handleInstanceVar(self, # Class variables can only be Attribute, so it's OK to cast because we used _maybeAttribute() above. obj = cast(Optional[model.Attribute], cls.contents.get(name)) if obj is None: - obj = self.builder.addAttribute(name=name, kind=None, parent=cls) - if annotation is None and expr is not None: - annotation = _infer_type(expr) - - obj.annotation = annotation - obj.setLineNumber(lineno) + self._setAttributeAnnotation(obj, annotation) - # Maybe an instance variable overrides a constant, - # so we check before setting the kind to INSTANCE_VARIABLE. - if obj.kind is model.DocumentableKind.CONSTANT: - self._warnsConstantReAssigmentInInstance(obj, lineno_offset=lineno-obj.linenumber) - else: - obj.kind = model.DocumentableKind.INSTANCE_VARIABLE - obj.value = expr - - self.builder.currentAttr = obj + obj.setLineNumber(lineno) + # undonditionnaly set the kind to ivar + obj.kind = model.DocumentableKind.INSTANCE_VARIABLE + self._storeAttrValue(obj, expr) + self._storeCurrentAttr(obj) def _handleAssignmentInClass(self, target: str, annotation: Optional[ast.expr], expr: Optional[ast.expr], - lineno: int + lineno: int, + augassign:Optional[ast.operator], ) -> None: cls = self.builder.current assert isinstance(cls, model.Class) if not _handleAliasing(cls, target, expr): - self._handleClassVar(target, annotation, expr, lineno) + self._handleClassVar(target, annotation, expr, lineno, augassign=augassign) def _handleDocstringUpdate(self, targetNode: ast.expr, @@ -689,17 +726,18 @@ def _handleAssignment(self, targetNode: ast.expr, annotation: Optional[ast.expr], expr: Optional[ast.expr], - lineno: int + lineno: int, + augassign:Optional[ast.operator]=None, ) -> None: if isinstance(targetNode, ast.Name): target = targetNode.id scope = self.builder.current if isinstance(scope, model.Module): - self._handleAssignmentInModule(target, annotation, expr, lineno) + self._handleAssignmentInModule(target, annotation, expr, lineno, augassign=augassign) elif isinstance(scope, model.Class): - if not self._handleOldSchoolMethodDecoration(target, expr): - self._handleAssignmentInClass(target, annotation, expr, lineno) - elif isinstance(targetNode, ast.Attribute): + if augassign or not self._handleOldSchoolMethodDecoration(target, expr): + self._handleAssignmentInClass(target, annotation, expr, lineno, augassign=augassign) + elif isinstance(targetNode, ast.Attribute) and not augassign: value = targetNode.value if targetNode.attr == '__doc__': self._handleDocstringUpdate(value, expr, lineno) @@ -728,6 +766,10 @@ def visit_Assign(self, node: ast.Assign) -> None: def visit_AnnAssign(self, node: ast.AnnAssign) -> None: annotation = unstring_annotation(node.annotation, self.builder.current) self._handleAssignment(node.target, annotation, node.value, node.lineno) + + def visit_AugAssign(self, node:ast.AugAssign) -> None: + self._handleAssignment(node.target, None, node.value, + node.lineno, augassign=node.op) def visit_Expr(self, node: ast.Expr) -> None: value = node.value @@ -904,7 +946,9 @@ def _handlePropertyDef(self, lineno: int ) -> model.Attribute: - attr = self.builder.addAttribute(name=node.name, kind=model.DocumentableKind.PROPERTY, parent=self.builder.current) + attr = self.builder.addAttribute(name=node.name, + kind=model.DocumentableKind.PROPERTY, + parent=self.builder.current) attr.setLineNumber(lineno) if docstring is not None: @@ -1014,59 +1058,6 @@ def __repr__(self) -> str: """ return '%s' % super().__repr__() - -def _infer_type(expr: ast.expr) -> Optional[ast.expr]: - """Infer an expression's type. - @param expr: The expression's AST. - @return: A type annotation, or None if the expression has no obvious type. - """ - try: - value: object = ast.literal_eval(expr) - except (ValueError, TypeError): - return None - else: - ann = _annotation_for_value(value) - if ann is None: - return None - else: - return ast.fix_missing_locations(ast.copy_location(ann, expr)) - -def _annotation_for_value(value: object) -> Optional[ast.expr]: - if value is None: - return None - name = type(value).__name__ - if isinstance(value, (dict, list, set, tuple)): - ann_elem = _annotation_for_elements(value) - if isinstance(value, dict): - ann_value = _annotation_for_elements(value.values()) - if ann_value is None: - ann_elem = None - elif ann_elem is not None: - ann_elem = ast.Tuple(elts=[ann_elem, ann_value]) - if ann_elem is not None: - if name == 'tuple': - ann_elem = ast.Tuple(elts=[ann_elem, ast.Ellipsis()]) - return ast.Subscript(value=ast.Name(id=name), - slice=ast.Index(value=ann_elem)) - return ast.Name(id=name) - -def _annotation_for_elements(sequence: Iterable[object]) -> Optional[ast.expr]: - names = set() - for elem in sequence: - ann = _annotation_for_value(elem) - if isinstance(ann, ast.Name): - names.add(ann.id) - else: - # Nested sequences are too complex. - return None - if len(names) == 1: - name = names.pop() - return ast.Name(id=name) - else: - # Empty sequence or no uniform type. - return None - - DocumentableT = TypeVar('DocumentableT', bound=model.Documentable) class ASTBuilder: diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 5677e5c68..36fa38293 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -6,7 +6,7 @@ import platform import sys from numbers import Number -from typing import Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union +from typing import Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union, cast from inspect import BoundArguments, Signature import ast @@ -402,4 +402,85 @@ def extract_docstring(node: ast.Str) -> Tuple[int, str]: - The docstring to be parsed, cleaned by L{inspect.cleandoc}. """ lineno = extract_docstring_linenum(node) - return lineno, inspect.cleandoc(node.s) \ No newline at end of file + return lineno, inspect.cleandoc(node.s) + + +def infer_type(expr: ast.expr) -> Optional[ast.expr]: + """Infer a literal expression's type. + @param expr: The expression's AST. + @return: A type annotation, or None if the expression has no obvious type. + """ + try: + value: object = ast.literal_eval(expr) + except (ValueError, TypeError): + return None + else: + ann = _annotation_for_value(value) + if ann is None: + return None + else: + return ast.fix_missing_locations(ast.copy_location(ann, expr)) + +def _annotation_for_value(value: object) -> Optional[ast.expr]: + if value is None: + return None + name = type(value).__name__ + if isinstance(value, (dict, list, set, tuple)): + ann_elem = _annotation_for_elements(value) + if isinstance(value, dict): + ann_value = _annotation_for_elements(value.values()) + if ann_value is None: + ann_elem = None + elif ann_elem is not None: + ann_elem = ast.Tuple(elts=[ann_elem, ann_value]) + if ann_elem is not None: + if name == 'tuple': + ann_elem = ast.Tuple(elts=[ann_elem, ast.Ellipsis()]) + return ast.Subscript(value=ast.Name(id=name), + slice=ast.Index(value=ann_elem)) + return ast.Name(id=name) + +def _annotation_for_elements(sequence: Iterable[object]) -> Optional[ast.expr]: + names = set() + for elem in sequence: + ann = _annotation_for_value(elem) + if isinstance(ann, ast.Name): + names.add(ann.id) + else: + # Nested sequences are too complex. + return None + if len(names) == 1: + name = names.pop() + return ast.Name(id=name) + else: + # Empty sequence or no uniform type. + return None + + +class Parentage(ast.NodeTransformer): + """ + Add C{parent} attribute to ast nodes instances. + """ + # stolen from https://stackoverflow.com/a/68845448 + parent: Optional[ast.AST] = None + + def visit(self, node: ast.AST) -> ast.AST: + setattr(node, 'parent', self.parent) + self.parent = node + node = super().visit(node) + if isinstance(node, ast.AST): + self.parent = getattr(node, 'parent') + return node + +def get_parents(node:ast.AST) -> Iterator[ast.AST]: + """ + Once nodes have the C{.parent} attribute with {Parentage}, use this function + to get a iterator on all parents of the given node up to the root module. + """ + def _yield_parents(n:Optional[ast.AST]) -> Iterator[ast.AST]: + if n: + yield n + p = cast(ast.AST, getattr(n, 'parent', None)) + yield from _yield_parents(p) + yield from _yield_parents(getattr(node, 'parent', None)) + diff --git a/pydoctor/epydoc/markup/_pyval_repr.py b/pydoctor/epydoc/markup/_pyval_repr.py index 7eae1918f..b1b8b4985 100644 --- a/pydoctor/epydoc/markup/_pyval_repr.py +++ b/pydoctor/epydoc/markup/_pyval_repr.py @@ -51,7 +51,7 @@ from pydoctor.epydoc.markup import DocstringLinker from pydoctor.epydoc.markup.restructuredtext import ParsedRstDocstring from pydoctor.epydoc.docutils import set_node_attributes, wbr, obj_reference, new_document -from pydoctor.astutils import node2dottedname, bind_args +from pydoctor.astutils import node2dottedname, bind_args, Parentage, get_parents def decode_with_backslashreplace(s: bytes) -> str: r""" @@ -111,21 +111,6 @@ def restore(self, mark: _MarkedColorizerState) -> List[nodes.Node]: del self.result[mark.length:] return trimmed -class _Parentage(ast.NodeTransformer): - """ - Add C{parent} attribute to ast nodes instances. - """ - # stolen from https://stackoverflow.com/a/68845448 - parent: Optional[ast.AST] = None - - def visit(self, node: ast.AST) -> ast.AST: - setattr(node, 'parent', self.parent) - self.parent = node - node = super().visit(node) - if isinstance(node, ast.AST): - self.parent = getattr(node, 'parent') - return node - # TODO: add support for comparators when needed. # _OperatorDelimitier is needed for: # - IfExp @@ -152,10 +137,14 @@ def __init__(self, colorizer: 'PyvalColorizer', state: _ColorizerState, self.marked = state.mark() # We use a hack to populate a "parent" attribute on AST nodes. - # See _Parentage class, applied in PyvalColorizer._colorize_ast() - parent_node: Optional[ast.AST] = getattr(node, 'parent', None) - - if parent_node: + # See astutils.Parentage class, applied in PyvalColorizer._colorize_ast() + try: + parent_node: ast.AST = next(get_parents(node)) + except StopIteration: + return + + # avoid needless parenthesis, since we now collect parents for every nodes + if isinstance(parent_node, (ast.expr, ast.keyword, ast.comprehension)): precedence = astor.op_util.get_op_precedence(node.op) if isinstance(parent_node, (ast.UnaryOp, ast.BinOp, ast.BoolOp)): parent_precedence = astor.op_util.get_op_precedence(parent_node.op) @@ -547,8 +536,10 @@ def _colorize_ast_constant(self, pyval: ast.AST, state: _ColorizerState) -> None def _colorize_ast(self, pyval: ast.AST, state: _ColorizerState) -> None: # Set nodes parent in order to check theirs precedences and add delimiters when needed. - if not getattr(pyval, 'parent', None): - _Parentage().visit(pyval) + try: + next(get_parents(pyval)) + except StopIteration: + Parentage().visit(pyval) if self._is_ast_constant(pyval): self._colorize_ast_constant(pyval, state) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 50dccc7b6..364b41e22 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -108,7 +108,7 @@ def annotation_from_attrib( return astutils.unstring_annotation(typ, ctx) default = args.arguments.get('default') if default is not None: - return astbuilder._infer_type(default) + return astutils.infer_type(default) return None class ModuleVisitor(extensions.ModuleVisitorExt): diff --git a/pydoctor/model.py b/pydoctor/model.py index 3933e03d2..e6d1ebd76 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -840,7 +840,7 @@ class FunctionOverload: class Attribute(Inheritable): kind: Optional[DocumentableKind] = DocumentableKind.ATTRIBUTE - annotation: Optional[ast.expr] + annotation: Optional[ast.expr] = None decorators: Optional[Sequence[ast.expr]] = None value: Optional[ast.expr] = None """ @@ -1448,22 +1448,37 @@ def fetchIntersphinxInventories(self, cache: CacheT) -> None: self.intersphinx.update(cache, url) def defaultPostProcess(system:'System') -> None: - # default post-processing includes: - # - Processing of subclasses - # - MRO computing. - # - Lookup of constructors - # - Checking whether the class is an exception - for cls in system.objectsOfType(Class): - + for cls in self.objectsOfType(Class): + # Initiate the MROs cls._init_mro() + # Lookup of constructors cls._init_constructors() - + + # Compute subclasses for b in cls.baseobjects: if b is not None: b.subclasses.append(cls) - + + # Checking whether the class is an exception if is_exception(cls): cls.kind = DocumentableKind.EXCEPTION + + for attrib in self.objectsOfType(Attribute): + _inherits_instance_variable_kind(attrib) + +def _inherits_instance_variable_kind(attr: Attribute) -> None: + """ + If any of the inherited members of a class variable is an instance variable, + then the subclass' class variable become an instance variable as well. + """ + if attr.kind is not DocumentableKind.CLASS_VARIABLE: + return + docsources = attr.docsources() + next(docsources) + for inherited in docsources: + if inherited.kind is DocumentableKind.INSTANCE_VARIABLE: + attr.kind = DocumentableKind.INSTANCE_VARIABLE + break def get_docstring( obj: Documentable diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index 903476f2f..b32a474f7 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -1829,9 +1829,9 @@ def __init__(**args): assert not captured @systemcls_param -def test_constant_override_in_instace_warns(systemcls: Type[model.System], capsys: CapSys) -> None: +def test_constant_override_in_instace(systemcls: Type[model.System], capsys: CapSys) -> None: """ - It warns when a constant is beeing re defined in instance. But it ignores it's value. + When an instance variable overrides a CONSTANT, it's flagged as INSTANCE_VARIABLE and no warning is raised. """ mod = fromText(''' class Clazz: @@ -1842,18 +1842,13 @@ def __init__(self, **args): ''', systemcls=systemcls, modname="mod") attr = mod.resolveName('Clazz.LANG') assert isinstance(attr, model.Attribute) - assert attr.kind == model.DocumentableKind.CONSTANT - assert attr.value is not None - assert ast.literal_eval(attr.value) == 'EN' - - captured = capsys.readouterr().out - assert "mod:6: Assignment to constant \"LANG\" inside an instance is ignored, this value will not be part of the docs.\n" == captured + assert attr.kind == model.DocumentableKind.INSTANCE_VARIABLE + assert not capsys.readouterr().out @systemcls_param -def test_constant_override_in_instace_warns2(systemcls: Type[model.System], capsys: CapSys) -> None: +def test_constant_override_in_instace_bis(systemcls: Type[model.System], capsys: CapSys) -> None: """ - It warns when a constant is beeing re defined in instance. But it ignores it's value. - Even if the actual constant definition is detected after the instance variable of the same name. + When an instance variable overrides a CONSTANT, it's flagged as INSTANCE_VARIABLE and no warning is raised. """ mod = fromText(''' class Clazz: @@ -1864,15 +1859,13 @@ def __init__(self, **args): ''', systemcls=systemcls, modname="mod") attr = mod.resolveName('Clazz.LANG') assert isinstance(attr, model.Attribute) - assert attr.kind == model.DocumentableKind.CONSTANT + assert attr.kind == model.DocumentableKind.INSTANCE_VARIABLE assert attr.value is not None assert ast.literal_eval(attr.value) == 'EN' - - captured = capsys.readouterr().out - assert "mod:5: Assignment to constant \"LANG\" inside an instance is ignored, this value will not be part of the docs.\n" == captured + assert not capsys.readouterr().out @systemcls_param -def test_constant_override_in_module_warns(systemcls: Type[model.System], capsys: CapSys) -> None: +def test_constant_override_in_module(systemcls: Type[model.System], capsys: CapSys) -> None: mod = fromText(''' """Mod.""" @@ -1883,12 +1876,10 @@ def test_constant_override_in_module_warns(systemcls: Type[model.System], capsys ''', systemcls=systemcls, modname="mod") attr = mod.resolveName('IS_64BITS') assert isinstance(attr, model.Attribute) - assert attr.kind == model.DocumentableKind.CONSTANT + assert attr.kind == model.DocumentableKind.VARIABLE assert attr.value is not None assert ast.literal_eval(attr.value) == True - - captured = capsys.readouterr().out - assert "mod:6: Assignment to constant \"IS_64BITS\" overrides previous assignment at line 4, the original value will not be part of the docs.\n" == captured + assert not capsys.readouterr().out @systemcls_param def test_constant_override_do_not_warns_when_defined_in_class_docstring(systemcls: Type[model.System], capsys: CapSys) -> None: @@ -1927,6 +1918,39 @@ def test_constant_override_do_not_warns_when_defined_in_module_docstring(systemc captured = capsys.readouterr().out assert not captured +@systemcls_param +def test_not_a_constant_module(systemcls: Type[model.System], capsys:CapSys) -> None: + """ + If the constant assignment has any kind of constraint or there are multiple assignments in the scope, + then it's not flagged as a constant. + """ + mod = fromText(''' + while False: + LANG = 'FR' + if True: + THING = 'EN' + OTHER = 1 + OTHER += 1 + E: typing.Final = 2 + E = 4 + LIST = [2.14] + LIST.insert(0,0) + ''', systemcls=systemcls) + assert mod.contents['LANG'].kind is model.DocumentableKind.VARIABLE + assert mod.contents['THING'].kind is model.DocumentableKind.VARIABLE + assert mod.contents['OTHER'].kind is model.DocumentableKind.VARIABLE + assert mod.contents['E'].kind is model.DocumentableKind.VARIABLE + + # all-caps mutables variables are flagged as constant: this is a trade-off + # in between our weeknesses in terms static analysis (that is we don't recognized list modifications) + # and our will to do the right thing and display constant values. + # This issue could be overcome by showing the value of variables with only one assigment no matter + # their kind and restrict the checks to immutable types for a attribute to be flagged as constant. + assert mod.contents['LIST'].kind is model.DocumentableKind.CONSTANT + + # we could warn when a constant is beeing overriden, but we don't: pydoctor is not a checker. + assert not capsys.readouterr().out + @systemcls_param def test__name__equals__main__is_skipped(systemcls: Type[model.System]) -> None: """ @@ -2397,6 +2421,287 @@ def __init__(self): mod = fromText(src, systemcls=systemcls) assert getConstructorsText(mod.contents['Animal']) == "Animal()" +@systemcls_param +def test_class_var_override(systemcls: Type[model.System]) -> None: + + src = '''\ + from number import Number + class Thing(object): + def __init__(self): + self.var: Number = 1 + class Stuff(Thing): + var:float + ''' + + mod = fromText(src, systemcls=systemcls, modname='mod') + var = mod.system.allobjects['mod.Stuff.var'] + assert var.kind == model.DocumentableKind.INSTANCE_VARIABLE + +@systemcls_param +def test_class_var_override_traverse_subclasses(systemcls: Type[model.System]) -> None: + + src = '''\ + from number import Number + class Thing(object): + def __init__(self): + self.var: Number = 1 + class _Stuff(Thing): + ... + class Stuff(_Stuff): + var:float + ''' + + mod = fromText(src, systemcls=systemcls, modname='mod') + var = mod.system.allobjects['mod.Stuff.var'] + assert var.kind == model.DocumentableKind.INSTANCE_VARIABLE + + src = '''\ + from number import Number + class Thing(object): + def __init__(self): + self.var: Optional[Number] = 0 + class _Stuff(Thing): + var = None + class Stuff(_Stuff): + var: float + ''' + + mod = fromText(src, systemcls=systemcls, modname='mod') + var = mod.system.allobjects['mod._Stuff.var'] + assert var.kind == model.DocumentableKind.INSTANCE_VARIABLE + + mod = fromText(src, systemcls=systemcls, modname='mod') + var = mod.system.allobjects['mod.Stuff.var'] + assert var.kind == model.DocumentableKind.INSTANCE_VARIABLE + +def test_class_var_override_attrs() -> None: + + systemcls = AttrsSystem + + src = '''\ + import attr + @attr.s + class Thing(object): + var = attr.ib() + class Stuff(Thing): + var: float + ''' + + mod = fromText(src, systemcls=systemcls, modname='mod') + var = mod.system.allobjects['mod.Stuff.var'] + assert var.kind == model.DocumentableKind.INSTANCE_VARIABLE + +@systemcls_param +def test_explicit_annotation_wins_over_inferred_type(systemcls: Type[model.System]) -> None: + """ + Explicit annotations are the preffered way of presenting the type of an attribute. + """ + src = '''\ + class Stuff(object): + thing: List[Tuple[Thing, ...]] + def __init__(self): + self.thing = [] + ''' + mod = fromText(src, systemcls=systemcls, modname='mod') + thing = mod.system.allobjects['mod.Stuff.thing'] + assert flatten_text(epydoc2stan.type2stan(thing)) == "List[Tuple[Thing, ...]]" #type:ignore + + src = '''\ + class Stuff(object): + thing = [] + def __init__(self): + self.thing: List[Tuple[Thing, ...]] = [] + ''' + mod = fromText(src, systemcls=systemcls, modname='mod') + thing = mod.system.allobjects['mod.Stuff.thing'] + assert flatten_text(epydoc2stan.type2stan(thing)) == "List[Tuple[Thing, ...]]" #type:ignore + +@systemcls_param +def test_explicit_inherited_annotation_looses_over_inferred_type(systemcls: Type[model.System]) -> None: + """ + Annotation are of inherited. + """ + src = '''\ + class _Stuff(object): + thing: List[Tuple[Thing, ...]] + class Stuff(_Stuff): + def __init__(self): + self.thing = [] + ''' + mod = fromText(src, systemcls=systemcls, modname='mod') + thing = mod.system.allobjects['mod.Stuff.thing'] + assert flatten_text(epydoc2stan.type2stan(thing)) == "list" #type:ignore + +@systemcls_param +def test_inferred_type_override(systemcls: Type[model.System]) -> None: + """ + The last visited value will be used to infer the type annotation + of an unnanotated attribute. + """ + src = '''\ + class Stuff(object): + thing = 1 + def __init__(self): + self.thing = (1,2) + ''' + mod = fromText(src, systemcls=systemcls, modname='mod') + thing = mod.system.allobjects['mod.Stuff.thing'] + assert flatten_text(epydoc2stan.type2stan(thing)) == "tuple[int, ...]" #type:ignore + +@systemcls_param +def test_inferred_type_is_not_propagated_to_subclasses(systemcls: Type[model.System]) -> None: + """ + Inferred type annotation should not be propagated to subclasses. + """ + src = '''\ + class _Stuff(object): + def __init__(self): + self.thing = [] + class Stuff(_Stuff): + def __init__(self, thing): + self.thing = thing + ''' + mod = fromText(src, systemcls=systemcls, modname='mod') + thing = mod.system.allobjects['mod.Stuff.thing'] + assert epydoc2stan.type2stan(thing) is None + + +@systemcls_param +def test_inherited_type_is_not_propagated_to_subclasses(systemcls: Type[model.System]) -> None: + """ + We can't repliably propage the annotations from one class to it's subclass because of + issue https://github.com/twisted/pydoctor/issues/295. + """ + src1 = '''\ + class _s:... + class _Stuff(object): + def __init__(self): + self.thing:_s = [] + ''' + src2 = '''\ + from base import _Stuff, _s + class Stuff(_Stuff): + def __init__(self, thing): + self.thing = thing + __all__=['Stuff', '_s'] + ''' + system = systemcls() + builder = system.systemBuilder(system) + builder.addModuleString(src1, 'base') + builder.addModuleString(src2, 'mod') + builder.buildModules() + thing = system.allobjects['mod.Stuff.thing'] + assert epydoc2stan.type2stan(thing) is None + +@systemcls_param +def test_augmented_assignment(systemcls: Type[model.System]) -> None: + mod = fromText(''' + var = 1 + var += 3 + ''', systemcls=systemcls) + attr = mod.contents['var'] + assert isinstance(attr, model.Attribute) + assert attr.value + assert astor.to_source(attr.value).strip() == '(1 + 3)' + +@systemcls_param +def test_augmented_assignment_in_class(systemcls: Type[model.System]) -> None: + mod = fromText(''' + class c: + var = 1 + var += 3 + ''', systemcls=systemcls) + attr = mod.contents['c'].contents['var'] + assert isinstance(attr, model.Attribute) + assert attr.value + assert astor.to_source(attr.value).strip() == '(1 + 3)' + + +@systemcls_param +def test_augmented_assignment_conditionnal_else_ignored(systemcls: Type[model.System]) -> None: + """ + The If.body branch is the only one in use. + """ + mod = fromText(''' + var = 1 + if something(): + var += 3 + else: + var += 4 + ''', systemcls=systemcls) + attr = mod.contents['var'] + assert isinstance(attr, model.Attribute) + assert attr.value + assert astor.to_source(attr.value).strip() == '(1 + 3)' + +@systemcls_param +def test_augmented_assignment_conditionnal_multiple_assignments(systemcls: Type[model.System]) -> None: + """ + The If.body branch is the only one in use, but several Ifs which have + theoritical exclusive conditions might be wrongly interpreted. + """ + mod = fromText(''' + var = 1 + if something(): + var += 3 + if not_something(): + var += 4 + ''', systemcls=systemcls) + attr = mod.contents['var'] + assert isinstance(attr, model.Attribute) + assert attr.value + assert astor.to_source(attr.value).strip() == '(1 + 3 + 4)' + +@systemcls_param +def test_augmented_assignment_instance_var(systemcls: Type[model.System]) -> None: + """ + Augmented assignments in instance var are not analyzed. + """ + mod = fromText(''' + class c: + def __init__(self, var): + self.var = 1 + self.var += var + ''') + attr = mod.contents['c'].contents['var'] + assert isinstance(attr, model.Attribute) + assert attr.value + assert astor.to_source(attr.value).strip() == '(1)' + +@systemcls_param +def test_augmented_assignment_not_suitable_for_inline_docstring(systemcls: Type[model.System]) -> None: + """ + Augmented assignments cannot have docstring attached. + """ + mod = fromText(''' + var = 1 + var += 1 + """ + this is not a docstring + """ + class c: + var = 1 + var += 1 + """ + this is not a docstring + """ + ''') + attr = mod.contents['var'] + assert not attr.docstring + attr = mod.contents['c'].contents['var'] + assert not attr.docstring + +@systemcls_param +def test_augmented_assignment_alone_is_not_documented(systemcls: Type[model.System]) -> None: + mod = fromText(''' + var += 1 + class c: + var += 1 + ''') + + assert 'var' not in mod.contents + assert 'var' not in mod.contents['c'].contents + @systemcls_param def test_typealias_unstring(systemcls: Type[model.System]) -> None: """ @@ -2413,4 +2718,5 @@ def test_typealias_unstring(systemcls: Type[model.System]) -> None: assert typealias.value with pytest.raises(StopIteration): # there is not Constant nodes in the type alias anymore - next(n for n in ast.walk(typealias.value) if isinstance(n, ast.Constant)) \ No newline at end of file + next(n for n in ast.walk(typealias.value) if isinstance(n, ast.Constant)) +