diff --git a/README.rst b/README.rst index 27745e6f1..935afe37d 100644 --- a/README.rst +++ b/README.rst @@ -196,7 +196,7 @@ pydoctor 22.7.0 * Improve the extensibility of pydoctor (`more infos on extensions `_) * Fix line numbers in reStructuredText xref warnings. * Add support for `twisted.python.deprecated` (this was originally part of Twisted's customizations). -* Add support for re-exporting it names imported from a wildcard import. +* Add support for re-exporting names imported from a wildcard import. pydoctor 22.5.1 ^^^^^^^^^^^^^^^ diff --git a/docs/epytext_demo/demo_epytext_module.py b/docs/epytext_demo/demo_epytext_module.py index 2763ccd2f..5f82e513f 100644 --- a/docs/epytext_demo/demo_epytext_module.py +++ b/docs/epytext_demo/demo_epytext_module.py @@ -6,7 +6,7 @@ from abc import ABC import math -from typing import overload, AnyStr, Dict, Generator, List, Union, Callable, Tuple, TYPE_CHECKING +from typing import overload, AnyStr, Dict, Generator, List, Union, Callable, Tuple, Sequence, Optional, Protocol, TYPE_CHECKING from somelib import SomeInterface import zope.interface import zope.schema @@ -32,6 +32,9 @@ This is also a constant, but annotated with typing.Final. """ +Interface = Protocol +"""Aliases are also documented.""" + @deprecated(Version("demo", "NEXT", 0, 0), replacement=math.prod) def demo_product_deprecated(x, y) -> float: # type: ignore return float(x * y) diff --git a/docs/restructuredtext_demo/demo_restructuredtext_module.py b/docs/restructuredtext_demo/demo_restructuredtext_module.py index 6f7da3cab..c404780cc 100644 --- a/docs/restructuredtext_demo/demo_restructuredtext_module.py +++ b/docs/restructuredtext_demo/demo_restructuredtext_module.py @@ -8,7 +8,7 @@ import math import zope.interface import zope.schema -from typing import overload, Callable, Sequence, Optional, AnyStr, Generator, Union, List, Dict, TYPE_CHECKING +from typing import overload, Protocol, Callable, Sequence, Optional, AnyStr, Generator, Union, List, Dict, TYPE_CHECKING from incremental import Version from twisted.python.deprecate import deprecated, deprecatedProperty @@ -30,6 +30,9 @@ This is also a constant, but annotated with typing.Final. """ +Interface = Protocol +"""Aliases are also documented.""" + @deprecated(Version("demo", "NEXT", 0, 0), replacement=math.prod) def demo_product_deprecated(x, y) -> float: # type: ignore return float(x * y) @@ -145,6 +148,7 @@ class DemoClass(ABC, _PrivateClass): .. versionchanged:: 1.2 Add `read_and_write_delete` property. """ + #FIXME: For some reason, the alias Demo do ont appear in the class page :/ def __init__(self, one: str, two: bytes) -> None: """ @@ -199,6 +203,12 @@ def read_and_write_delete(self) -> None: """ This is a docstring for deleter. """ + pass + + ro = read_only + rw = read_and_write + rwd = read_and_write_delete + class IContact(zope.interface.Interface): """ @@ -215,3 +225,6 @@ class IContact(zope.interface.Interface): def send_email(text: str) -> None: pass + +_Demo = _PrivateClass +Demo = DemoClass diff --git a/mypy.ini b/mypy.ini index 854f7a6be..630c866eb 100644 --- a/mypy.ini +++ b/mypy.ini @@ -14,6 +14,10 @@ warn_unused_ignores=True plugins=mypy_zope:plugin + +exclude = (?x)( + ^pydoctor\/test\/testpackages\/ + ) # The following modules are currently only partially annotated: [mypy-pydoctor.test.test_napoleon_docstring] diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index 1df74cd9e..3079058c6 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -3,6 +3,7 @@ import ast import sys +import builtins from functools import partial from inspect import Parameter, Signature @@ -19,6 +20,7 @@ is__name__equals__main__, unstring_annotation, upgrade_annotation, iterassign, extract_docstring_linenum, infer_type, get_parents, get_docstring_node, unparse, NodeVisitor, Parentage, Str) +_builtins_names = set(dir(builtins)) def parseFile(path: Path) -> ast.Module: """Parse the contents of a Python source file.""" @@ -44,26 +46,8 @@ def _maybeAttribute(cls: model.Class, name: str) -> bool: return obj is None or isinstance(obj, model.Attribute) -def _handleAliasing( - ctx: model.CanContainImportsDocumentable, - target: str, - expr: Optional[ast.expr] - ) -> bool: - """If the given expression is a name assigned to a target that is not yet - in use, create an alias. - @return: L{True} iff an alias was created. - """ - if target in ctx.contents: - return False - full_name = node2fullname(expr, ctx) - if full_name is None: - return False - ctx._localNameToFullName_map[target] = full_name - return True - - -_CONTROL_FLOW_BLOCKS:Tuple[Type[ast.stmt],...] = (ast.If, ast.While, ast.For, ast.Try, - ast.AsyncFor, ast.With, ast.AsyncWith) +_LOOP_BLOCKS: Tuple[Type[ast.stmt],...] = (ast.While, ast.For, ast.AsyncFor,) +_CONTROL_FLOW_BLOCKS: Tuple[Type[ast.stmt],...] = (ast.If, ast.Try, ast.With, ast.AsyncWith, *_LOOP_BLOCKS) """ AST types that introduces a new control flow block, potentially conditionnal. """ @@ -90,32 +74,55 @@ def is_constant(obj: model.Attribute, return obj.name.isupper() or is_using_typing_final(annotation, obj) return False +def is_alias(obj: model.Attribute, + annotation:Optional[ast.expr], + value: Optional[ast.expr]) -> bool: + """ + Detect if the given assignment is an alias. + + This is very similar to L{is_constant} except that: + - the value must be only composed by L{ast.Attribute} and L{ast.Name} instances. + - the attribute must not be a type alias, in which case it will be flagged as a type alias instead. + - conditional blocks are allowed in node ancestors, just no loops. + - it may be overriden + """ + if value and node2dottedname(value): + if not any(isinstance(n, _LOOP_BLOCKS) for n in get_parents(value)): + return not _is_typealias(obj, annotation, value) + return False + +def _is_typevar(ob: model.Documentable, + annotation:Optional[ast.expr], + value: ast.expr | None) -> bool: + if value is not None: + if isinstance(value, ast.Call) and \ + node2fullname(value.func, ob) in ('typing.TypeVar', + 'typing_extensions.TypeVar', + 'typing.TypeVarTuple', + 'typing_extensions.TypeVarTuple'): + return True + return False + +def _is_typealias(ob: model.Documentable, + annotation:Optional[ast.expr], + value: ast.expr | None) -> bool: + """ + Return C{True} if the Attribute is a type alias. + """ + if value is not None: + if is_using_annotations(annotation, ('typing.TypeAlias', + 'typing_extensions.TypeAlias'), ob): + return True + if is_typing_annotation(value, ob): + return True + return False + + class TypeAliasVisitorExt(extensions.ModuleVisitorExt): """ This visitor implements the handling of type aliases and type variables. """ - def _isTypeVariable(self, ob: model.Attribute) -> bool: - if ob.value is not None: - if isinstance(ob.value, ast.Call) and \ - node2fullname(ob.value.func, ob) in ('typing.TypeVar', - 'typing_extensions.TypeVar', - 'typing.TypeVarTuple', - 'typing_extensions.TypeVarTuple'): - return True - return False - def _isTypeAlias(self, ob: model.Attribute) -> bool: - """ - Return C{True} if the Attribute is a type alias. - """ - if ob.value is not None: - if is_using_annotations(ob.annotation, ('typing.TypeAlias', - 'typing_extensions.TypeAlias'), ob): - return True - if is_typing_annotation(ob.value, ob.parent): - return True - return False - def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: current = self.visitor.builder.current for dottedname in iterassign(node): @@ -125,13 +132,13 @@ def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: return if not isinstance(attr, model.Attribute): return - if self._isTypeAlias(attr) is True: + if _is_typealias(attr, attr.annotation, attr.value) is True: attr.kind = model.DocumentableKind.TYPE_ALIAS # unstring type aliases attr.value = upgrade_annotation(unstring_annotation( # this cast() is safe because _isTypeAlias() return True only if value is not None cast(ast.expr, attr.value), attr, section='type alias'), attr, section='type alias') - elif self._isTypeVariable(attr) is True: + elif _is_typevar(attr, attr.annotation, attr.value) is True: # TODO: unstring bound argument of type variables attr.kind = model.DocumentableKind.TYPE_VARIABLE @@ -231,6 +238,11 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None: str_base = '.'.join(node2dottedname(name_node) or \ # Fallback on unparse() if the expression is unknown by node2dottedname(). [unparse(base_node).strip()]) + + # Special case builtins names so they are preceeded with 'builtins'. + if str_base in _builtins_names and not parent.isNameDefined(str_base): + # A non-shadowed builtins + str_base = f'builtins.{str_base}' # Store the base as string and as ast.expr in rawbases list. rawbases += [(str_base, base_node)] @@ -239,6 +251,7 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None: # if we can't resolve it now, it most likely mean that there are # import cycles (maybe in TYPE_CHECKING blocks). # None bases will be re-resolved in post-processing. + expandbase = parent.expandName(str_base) baseobj = self.system.objForFullName(expandbase) @@ -327,11 +340,11 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> None: assert modname is not None if node.names[0].name == '*': - self._importAll(modname) + self._importAll(modname, lineno=node.lineno) else: - self._importNames(modname, node.names) + self._importNames(modname, node.names, lineno=node.lineno) - def _importAll(self, modname: str) -> None: + def _importAll(self, modname: str, lineno:int) -> None: """Handle a C{from import *} statement.""" mod = self.system.getProcessedModule(modname) @@ -358,15 +371,15 @@ def _importAll(self, modname: str) -> None: exports = self._getCurrentModuleExports() # Add imported names to our module namespace. - assert isinstance(self.builder.current, model.CanContainImportsDocumentable) - _localNameToFullName = self.builder.current._localNameToFullName_map + current = self.builder.current + assert isinstance(current, model.CanContainImportsDocumentable) + _localNameToFullName = current._localNameToFullName_map expandName = mod.expandName for name in names: - if self._handleReExport(exports, name, name, mod) is True: continue - - _localNameToFullName[name] = expandName(name) + _localNameToFullName[name] = model.ImportAlias(self.system, name, + alias=expandName(name), parent=current, linenumber=lineno) def _getCurrentModuleExports(self) -> Collection[str]: # Fetch names to export. @@ -382,35 +395,52 @@ def _getCurrentModuleExports(self) -> Collection[str]: def _handleReExport(self, curr_mod_exports:Collection[str], origin_name:str, as_name:str, - origin_module:model.Module) -> bool: + origin_module:Union[model.Module, str]) -> bool: """ Move re-exported objects into current module. + @param origin_module: None if the module is unknown to this system. @returns: True if the imported name has been sucessfully re-exported. """ # Move re-exported objects into current module. current = self.builder.current - modname = origin_module.fullName() + if isinstance(origin_module, model.Module): + modname = origin_module.fullName() + known_module = True + else: + modname = origin_module + known_module = False if as_name in curr_mod_exports: # In case of duplicates names, we can't rely on resolveName, # So we use content.get first to resolve non-alias names. - ob = origin_module.contents.get(origin_name) or origin_module.resolveName(origin_name) - if ob is None: - current.report("cannot resolve re-exported name :" - f'{modname}.{origin_name}', thresh=1) + if known_module: + assert isinstance(origin_module, model.Module) + ob = origin_module.contents.get(origin_name) or origin_module.resolveName(origin_name) + if ob is None: + current.report("cannot resolve re-exported name", + f'{modname}.{origin_name}', thresh=1) + else: + if origin_module.all is None or origin_name not in origin_module.all: + self.system.msg( + "astbuilder", + "moving %r into %r" % (ob.fullName(), current.fullName()) + ) + # Must be a Module since the exports is set to an empty list if it's not. + assert isinstance(current, model.Module) + ob.reparent(current, as_name) + return True else: - if origin_module.all is None or origin_name not in origin_module.all: - self.system.msg( - "astbuilder", - "moving %r into %r" % (ob.fullName(), current.fullName()) - ) - # Must be a Module since the exports is set to an empty list if it's not. - assert isinstance(current, model.Module) - ob.reparent(current, as_name) - return True + # return False + # re-export names that are not part of the current system with an alias, this should be done in post-processing + attr = self.builder.addAttribute(name=as_name, kind=model.DocumentableKind.ALIAS, parent=current) + attr.alias = f'{modname}.{origin_name}' + # This is only for the HTML repr + attr.value=ast.Name(attr.alias, ast.Load()) # passing ctx is required for python 3.6 + return True + return False - def _importNames(self, modname: str, names: Iterable[ast.alias]) -> None: + def _importNames(self, modname: str, names: Iterable[ast.alias], lineno:int) -> None: """Handle a C{from import } statement.""" # Process the module we're importing from. @@ -426,14 +456,20 @@ def _importNames(self, modname: str, names: Iterable[ast.alias]) -> None: orgname, asname = al.name, al.asname if asname is None: asname = orgname + + # if self._handleReExport(exports, orgname, asname, mod or modname) is True: + # continue + # If we're importing from a package, make sure imported modules # are processed (getProcessedModule() ignores non-modules). if isinstance(mod, model.Package): self.system.getProcessedModule(f'{modname}.{orgname}') - if mod is not None and self._handleReExport(exports, orgname, asname, mod) is True: - continue - _localNameToFullName[asname] = f'{modname}.{orgname}' + _localNameToFullName[asname] = model.ImportAlias(self.system, asname, + alias=f'{modname}.{orgname}', parent=current, linenumber=lineno) + + if self._handleReExport(exports, orgname, asname, mod or modname) is True: + continue def visit_Import(self, node: ast.Import) -> None: """Process an import statement. @@ -448,16 +484,24 @@ def visit_Import(self, node: ast.Import) -> None: (dotted_name, as_name) where as_name is None if there was no 'as foo' part of the statement. """ - if not isinstance(self.builder.current, model.CanContainImportsDocumentable): + current = self.builder.current + if not isinstance(current, model.CanContainImportsDocumentable): # processing import statement in odd context return - _localNameToFullName = self.builder.current._localNameToFullName_map + _localNameToFullName = current._localNameToFullName_map for al in node.names: targetname, asname = al.name, al.asname if asname is None: # we're keeping track of all defined names asname = targetname = targetname.split('.')[0] - _localNameToFullName[asname] = targetname + + _localNameToFullName[asname] = model.ImportAlias(self.system, asname, + alias=targetname, parent=current, linenumber=node.lineno) + + # fullname, asname = al.name, al.asname + # if asname is not None: + # _localNameToFullName[asname] = model.ImportAlias(self.system, asname, + # alias=fullname, parent=current, linenumber=node.lineno) def _handleOldSchoolMethodDecoration(self, target: str, expr: Optional[ast.expr]) -> bool: if not isinstance(expr, ast.Call): @@ -546,6 +590,43 @@ def _storeCurrentAttr(self, obj:model.Attribute, self.builder.currentAttr = obj else: self.builder.currentAttr = None + + @classmethod + def _handleAlias(cls, obj:model.Attribute, + annotation: ast.expr | None, + value:Optional[ast.expr], + lineno:int, + defaultKind:model.DocumentableKind) -> None: + """ + Must be called after obj.setLineNumber() to have the right line number in the warning. + + Create or update an alias. + """ + if is_alias(obj, annotation, value): + + # This will be used for HTML repr of the alias. + obj.value = value + dottedname = node2dottedname(value) + assert dottedname is not None + name = '.'.join(dottedname) + + defs = obj.getDefinitions(name, before=obj.linenumber) + # Store the alias value as string now, + # this avoids doing it in _resolveAlias(). + if (name in _builtins_names and not defs): + # it is a non-shadowed builtins + name = f'builtins.{name}' + elif defs: + # let's try to resolve the local name now + d = defs[-1] + fullname = d.alias if isinstance(d, model.ImportAlias) else d.fullName() + name = fullname + + obj.kind = model.DocumentableKind.ALIAS + obj.alias = name + + elif obj.kind is model.DocumentableKind.ALIAS: + obj.kind = defaultKind def _handleModuleVar(self, target: str, @@ -587,6 +668,8 @@ def _handleModuleVar(self, self._handleConstant(obj, annotation, expr, lineno, model.DocumentableKind.VARIABLE) + self._handleAlias(obj, annotation, expr, lineno, + model.DocumentableKind.VARIABLE) self._storeAttrValue(obj, expr, augassign) self._storeCurrentAttr(obj, augassign) @@ -599,8 +682,7 @@ def _handleAssignmentInModule(self, ) -> None: module = self.builder.current assert isinstance(module, model.Module) - if not _handleAliasing(module, target, expr): - self._handleModuleVar(target, annotation, expr, lineno, augassign=augassign) + self._handleModuleVar(target, annotation, expr, lineno, augassign=augassign) def _handleClassVar(self, name: str, @@ -631,6 +713,8 @@ def _handleClassVar(self, self._handleConstant(obj, annotation, expr, lineno, model.DocumentableKind.CLASS_VARIABLE) + self._handleAlias(obj, annotation, expr, lineno, + model.DocumentableKind.CLASS_VARIABLE) self._storeAttrValue(obj, expr, augassign) self._storeCurrentAttr(obj, augassign) @@ -671,8 +755,7 @@ def _handleAssignmentInClass(self, ) -> None: cls = self.builder.current assert isinstance(cls, model.Class) - if not _handleAliasing(cls, target, expr): - self._handleClassVar(target, annotation, expr, lineno, augassign=augassign) + self._handleClassVar(target, annotation, expr, lineno, augassign=augassign) def _handleDocstringUpdate(self, targetNode: ast.expr, @@ -743,6 +826,7 @@ def _handleAssignment(self, self._handleDocstringUpdate(value, expr, lineno) elif isinstance(value, ast.Name) and value.id == 'self': self._handleInstanceVar(targetNode.attr, annotation, expr, lineno) + # TODO: Fix https://github.com/twisted/pydoctor/issues/13 def visit_Assign(self, node: ast.Assign) -> None: lineno = node.lineno @@ -1195,6 +1279,7 @@ def parseString(self, py_string:str, ctx: model.Module) -> Optional[ast.Module]: model.System.defaultBuilder = ASTBuilder + def findModuleLevelAssign(mod_ast: ast.Module) -> Iterator[Tuple[str, ast.Assign]]: """ Find module level Assign. diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 850414c05..165237c12 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -10,6 +10,7 @@ from typing import Any, Callable, Collection, Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union, cast from inspect import BoundArguments, Signature import ast +from numbers import Number if sys.version_info >= (3, 9): from ast import unparse as _unparse @@ -115,10 +116,15 @@ def node2dottedname(node: Optional[ast.AST]) -> Optional[List[str]]: parts.reverse() return parts + def node2fullname(expr: Optional[ast.AST], ctx: model.Documentable | None = None, *, expandName:Callable[[str], str] | None = None) -> Optional[str]: + """ + Returns the expanded name of this AST expression if C{expr} is a name, or C{None}. + A name is an expression only composed by `ast.Name` and `ast.Attribute` nodes. + """ if expandName is None: if ctx is None: raise TypeError('this function takes exactly two arguments') @@ -145,8 +151,6 @@ def bind_args(sig: Signature, call: ast.Call) -> BoundArguments: } return sig.bind(*call.args, **kwargs) - - if sys.version_info[:2] >= (3, 8): # Since Python 3.8 "foo" is parsed as ast.Constant. def get_str_value(expr:ast.expr) -> Optional[str]: diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index eda352d1a..84d5ea85d 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -703,6 +703,10 @@ def _get_parsed_summary(obj: model.Documentable) -> Tuple[Optional[model.Documen return (source, obj.parsed_summary) if source is None: + # if obj.kind is model.DocumentableKind.ALIAS: + # assert isinstance(obj, model.Attribute) + # # Aliases are generally not documented, so we never mark them as "undocumented", we simply link the object. + # return Tag('', children=format_alias_value(obj).children) summary_parsed_doc: ParsedDocstring = ParsedStanOnly(format_undocumented(obj)) else: # Tell mypy that if we found a docstring, we also have its source. @@ -790,7 +794,9 @@ def format_docstring(obj: model.Documentable) -> Tag: ret: Tag = tags.div if source is None: - ret(tags.p(class_='undocumented')("Undocumented")) + # Aliases are generally not documented, so we never mark them as "undocumented". + if obj.kind is not model.DocumentableKind.ALIAS: + ret(tags.p(class_='undocumented')("Undocumented")) else: assert obj.parsed_docstring is not None, "ensure_parsed_docstring() did not do it's job" stan = safe_to_stan(obj.parsed_docstring, source.docstring_linker, source, fallback=format_docstring_fallback) @@ -967,6 +973,7 @@ def format_kind(kind: model.DocumentableKind, plural: bool = False) -> str: model.DocumentableKind.VARIABLE : 'Variable', model.DocumentableKind.SCHEMA_FIELD : 'Attribute', model.DocumentableKind.CONSTANT : 'Constant', + model.DocumentableKind.ALIAS : 'Alias', model.DocumentableKind.EXCEPTION : 'Exception', model.DocumentableKind.TYPE_ALIAS : 'Type Alias', model.DocumentableKind.TYPE_VARIABLE : 'Type Variable', @@ -974,6 +981,7 @@ def format_kind(kind: model.DocumentableKind, plural: bool = False) -> str: plurals = { model.DocumentableKind.CLASS : 'Classes', model.DocumentableKind.PROPERTY : 'Properties', + model.DocumentableKind.ALIAS : 'Aliases', model.DocumentableKind.TYPE_ALIAS : 'Type Aliases', } if plural: @@ -1017,6 +1025,14 @@ def format_constant_value(obj: model.Attribute) -> "Flattenable": rows = list(_format_constant_value(obj)) return tags.table(class_='valueTable')(*rows) +def format_alias_value(obj: model.Attribute) -> Tag: + if isinstance(obj.resolved_alias, model.Documentable): + # TODO: contextualize the name in the context of the module/class, currently this always shows the fullName of the object. + alias = tags.code(taglink(obj.resolved_alias, obj.page_object.url)) + else: + alias = colorize_inline_pyval(obj.value).to_stan(obj.parent.docstring_linker) + return tags.p(tags.em("Alias to ", alias)) + def _split_indentifier_parts_on_case(indentifier:str) -> List[str]: def split(text:str, sep:str) -> List[str]: diff --git a/pydoctor/linker.py b/pydoctor/linker.py index a36949339..4683eace7 100644 --- a/pydoctor/linker.py +++ b/pydoctor/linker.py @@ -68,6 +68,11 @@ def __init__(self, obj: 'model.Documentable') -> None: self._init_obj = obj self._page_object: Optional['model.Documentable'] = obj.page_object + def debug(self, msg:str, lineno:int): + if self.reporting_obj is None: + return + self.reporting_obj.report(msg, 'resolve_identifier_xref', lineno, thresh=5) + @property def obj(self) -> 'model.Documentable': """ @@ -103,16 +108,34 @@ def switch_context(self, ob:Optional['model.Documentable']) -> Iterator[None]: def look_for_name(self, name: str, candidates: Iterable['model.Documentable'], - lineno: int + lineno: int, + look_for_imports: bool = False, ) -> Optional['model.Documentable']: - part0 = name.split('.')[0] - potential_targets = [] + self.debug(f'Linker looks for name {name!r} in several candidates...', lineno) + is_dotted_name = '.' in name + part0 = name.split('.')[0] if is_dotted_name else name + potential_targets: list[model.Documentable] = [] + potential_expanded_names: set[str] = set() for src in candidates: - if part0 not in src.contents: + # First look for objects that are not imports and then include imports if nothing was found, + name_defined = src.isNameDefined(part0) if look_for_imports else part0 in src.contents + if not name_defined: continue - target = src.resolveName(name) - if target is not None and target not in potential_targets: + # Emulates resolveName() + expanded_target = src.expandName(name) + target = src.system.objForFullName(expanded_target) + + if target is None and not is_dotted_name: + # replace an alias with its definition but use the alias is we fail to resolve it + # ignore aliases that point to a definition already in the collection + target = src.contents.get(name) + + + self.debug(f'Linker finds {part0} in {src} resolving name into {target}', lineno) + if target is not None and target not in potential_targets and expanded_target not in potential_expanded_names: potential_targets.append(target) + potential_expanded_names.add(expanded_target) + if len(potential_targets) == 1: return potential_targets[0] elif len(potential_targets) > 1 and self.reporting_obj: @@ -121,6 +144,9 @@ def look_for_name(self, name, ', '.join(ob.fullName() for ob in potential_targets)), 'resolve_identifier_xref', lineno) + elif not look_for_imports: + return self.look_for_name(name, candidates, lineno, look_for_imports=True) + return None def look_for_intersphinx(self, name: str) -> Optional[str]: @@ -186,16 +212,26 @@ def _resolve_identifier_xref(self, # Check if 'identifier' is the fullName of an object. target = self.obj.system.objForFullName(identifier) if target is not None: + self.debug(f'Linker found an exact full name match for {identifier!r}', lineno) return target - + else: + self.debug(f'Linker did not found an exact full name match for {identifier!r}', lineno) + # Check if the fullID exists in an intersphinx inventory. fullID = self.obj.expandName(identifier) + self.debug(f'Linker expands name {identifier!r} into {fullID!r} in the context of {self.obj}', lineno) + target_url = self.look_for_intersphinx(fullID) if not target_url: # FIXME: https://github.com/twisted/pydoctor/issues/125 # expandName is unreliable so in the case fullID fails, we # try our luck with 'identifier'. target_url = self.look_for_intersphinx(identifier) + if target_url: + self.debug(f'Linker did not found an intersphinx entry for {fullID!r}, but {identifier!r} worked', lineno) + else: + self.debug(f'Linker found an intersphinx entry for {fullID!r}', lineno) + if target_url: return target_url @@ -207,11 +243,13 @@ def _resolve_identifier_xref(self, # to an object by Python name resolution in each context. src: Optional['model.Documentable'] = self.obj while src is not None: - target = src.resolveName(identifier) + target = src.contents.get(identifier) or src.resolveName(identifier) if target is not None: + self.debug(f'Linker found an parent object of {self.obj} ({src}) in which the name {identifier!r} resolves to {target}', lineno) return target src = src.parent - + + self.debug(f'Linker did not found any parent object in which the name {identifier!r} is defined, continuing with uncle objects', lineno) # Walk up the object tree again and see if 'identifier' refers to an # object in an "uncle" object. (So if p.m1 has a class C, the # docstring for p.m2 can say L{C} to refer to the class in m1). @@ -220,15 +258,17 @@ def _resolve_identifier_xref(self, while src is not None: target = self.look_for_name(identifier, src.contents.values(), lineno) if target is not None: + self.debug(f'Linker found a uncle object of {self.obj} in which the name {identifier!r} resolves to {target}', lineno) return target src = src.parent + self.debug(f'Linker did not found any uncle object in the context of {self.obj} in which the name {identifier!r} is defined, continuing with searching all modules', lineno) # Examine every module and package in the system and see if 'identifier' # names an object in each one. Again, if more than one object is # found, complain. target = self.look_for_name( # System.objectsOfType now supports passing the type as string. - identifier, self.obj.system.objectsOfType('pydoctor.model.Module'), lineno) + identifier, self.obj.system.objectsOfType(self.obj.system.Module), lineno) if target is not None: return target diff --git a/pydoctor/model.py b/pydoctor/model.py index 425727ebb..04d177385 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -9,6 +9,8 @@ import abc import ast +from itertools import chain +from operator import attrgetter, itemgetter import attr from collections import defaultdict import datetime @@ -17,11 +19,11 @@ import sys import textwrap import types -from enum import Enum +from enum import Enum, IntEnum from inspect import signature, Signature from pathlib import Path from typing import ( - TYPE_CHECKING, Any, Collection, Dict, Iterator, List, Mapping, Callable, + TYPE_CHECKING, Any, Callable, Collection, Dict, Iterable, Iterator, List, Mapping, Optional, Sequence, Set, Tuple, Type, TypeVar, Union, cast, overload ) from urllib.parse import quote @@ -34,9 +36,11 @@ if TYPE_CHECKING: from typing_extensions import Literal, Protocol from pydoctor.astbuilder import ASTBuilder, DocumentableT + from pydoctor.names import _IndirectionT else: Literal = {True: bool, False: bool} ASTBuilder = Protocol = object + _IndirectionT = object # originally when I started to write pydoctor I had this idea of a big @@ -53,12 +57,6 @@ # Functions can't contain anything. -_string_lineno_is_end = sys.version_info < (3,8) \ - and platform.python_implementation() != 'PyPy' -"""True iff the 'lineno' attribute of an AST string node points to the last -line in the string, rather than the first line. -""" - class LineFromAst(int): "Simple L{int} wrapper for linenumbers coming from ast analysis." @@ -77,7 +75,6 @@ class ProcessingState(Enum): PROCESSING = 1 PROCESSED = 2 - class PrivacyClass(Enum): """L{Enum} containing values indicating how private an object should be. @@ -107,6 +104,7 @@ class DocumentableKind(Enum): STATIC_METHOD = 600 METHOD = 500 FUNCTION = 400 + ALIAS = 320 CONSTANT = 310 TYPE_VARIABLE = 306 TYPE_ALIAS = 305 @@ -117,6 +115,27 @@ class DocumentableKind(Enum): PROPERTY = 150 VARIABLE = 100 + +class ImportAlias: + """ + Imports are not documentable, but share bits of the interface. + """ + + def __init__(self, system: 'System', + name: str, alias:str, + parent: 'CanContainImportsDocumentable', + linenumber:int): + + self.system = system + self.name = name + self.parent = parent + self.linenumber = linenumber + self.alias: Optional[str] = alias + + def fullName(self) -> str: + return f'{self.parent.fullName()}.{self.name}' + + class Documentable: """An object that can be documented. @@ -138,6 +157,8 @@ class Documentable: documentation_location = DocLocation.OWN_PAGE """Page location where we are documented.""" + _RESOLVE_ALIAS_MAX_RECURSE = 100 + def __init__( self, system: 'System', name: str, parent: Optional['Documentable'] = None, @@ -150,10 +171,12 @@ def __init__( self.parent = parent self.parentMod: Optional[Module] = None self.source_path: Optional[Path] = source_path + self.extra_info: List[ParsedDocstring] = [] """ A list to store extra informations about this documentable, as L{ParsedDocstring}. """ + self.setup() @property @@ -162,6 +185,11 @@ def doctarget(self) -> 'Documentable': def setup(self) -> None: self.contents: Dict[str, Documentable] = {} + self.aliases: List[Documentable] = [] + """ + Aliases to this object. + Computed at the time of post-procesing. + """ self._linker: Optional['linker.DocstringLinker'] = None def setDocstring(self, node: astutils.Str) -> None: @@ -289,7 +317,10 @@ def reparent(self, new_parent: 'Module', new_name: str) -> None: self.name = new_name self._handle_reparenting_post() del old_parent.contents[old_name] - old_parent._localNameToFullName_map[old_name] = self.fullName() + # We could add a special alias insead of using _localNameToFullName_map, + # this would allow to track the original location of the documentable. + old_parent._localNameToFullName_map[old_name] = ImportAlias(self.system, old_name, + alias=self.fullName(), parent=old_parent, linenumber=self.linenumber) new_parent.contents[new_name] = self self._handle_reparenting_post() @@ -302,62 +333,26 @@ def _handle_reparenting_post(self) -> None: self.system.allobjects[self.fullName()] = self for o in self.contents.values(): o._handle_reparenting_post() - - def _localNameToFullName(self, name: str) -> str: - raise NotImplementedError(self._localNameToFullName) - - def isNameDefined(self, name:str) -> bool: - """ - Is the given name defined in the globals/locals of self-context? - Only the first name of a dotted name is checked. - Returns True iff the given name can be loaded without raising `NameError`. + def expandName(self, name: str, indirections:Any=None) -> str: + """ + See L{names.expandName} """ + from pydoctor import names + return names.expandName(self, name, indirections) + + def isNameDefined(self, name: str, before:int|None=None) -> bool: raise NotImplementedError(self.isNameDefined) - def expandName(self, name: str) -> str: - """Return a fully qualified name for the possibly-dotted `name`. - - To explain what this means, consider the following modules: - - mod1.py:: - - from external_location import External - class Local: - pass - - mod2.py:: - - from mod1 import External as RenamedExternal - import mod1 as renamed_mod - class E: - pass + def getDefinitions(self, name: str, before:int|None=None) -> list[Documentable | ImportAlias]: + """ + Find all registered definitions of the given name in the context of C{self}. + The definitions are sorted by source linenumber. - In the context of mod2.E, expandName("RenamedExternal") should be - "external_location.External" and expandName("renamed_mod.Local") - should be "mod1.Local". """ - parts = name.split('.') - obj: Documentable = self - for i, p in enumerate(parts): - full_name = obj._localNameToFullName(p) - if full_name == p and i != 0: - # The local name was not found. - # If we're looking at a class, we try our luck with the inherited members - if isinstance(obj, Class): - inherited = obj.find(p) - if inherited: - full_name = inherited.fullName() - if full_name == p: - # We don't have a full name - # TODO: Instead of returning the input, _localNameToFullName() - # should probably either return None or raise LookupError. - full_name = f'{obj.fullName()}.{p}' - break - nxt = self.system.objForFullName(full_name) - if nxt is None: - break - obj = nxt - return '.'.join([full_name] + parts[i + 1:]) + @param before: A linenumber to filter all definitions after this point. + Only works for the locals since upper scope definitions are executed before. + """ + raise NotImplementedError(self.getDefinitions) def expandAnnotationName(self, name: str) -> str: """ @@ -371,9 +366,12 @@ def expandAnnotationName(self, name: str) -> str: return self.module.expandName(name) def resolveName(self, name: str) -> Optional['Documentable']: - """Return the object named by "name" (using Python's lookup rules) in - this context, if any is known to pydoctor.""" - return self.system.objForFullName(self.expandName(name)) + """ + Return the object named by "name" (using Python's lookup rules) in + this context, if any is known to pydoctor. + """ + obj = self.system.objForFullName(self.expandName(name)) + return obj @property def privacyClass(self) -> PrivacyClass: @@ -455,19 +453,34 @@ def docstring_linker(self) -> 'linker.DocstringLinker': class CanContainImportsDocumentable(Documentable): def setup(self) -> None: super().setup() - self._localNameToFullName_map: Dict[str, str] = {} + self._localNameToFullName_map: Dict[str, ImportAlias] = {} - def isNameDefined(self, name: str) -> bool: + def isNameDefined(self, name: str, before:int|None=None) -> bool: name = name.split('.')[0] + return bool(self.getDefinitions(name, before)) + + def getDefinitions(self, name: str, before:int|None=None) -> list[Documentable | ImportAlias]: + defs: Iterator[Documentable | ImportAlias] = iter([]) if name in self.contents: - return True + defs = chain(defs, [self.contents[name]]) if name in self._localNameToFullName_map: - return True - if not isinstance(self, Module): - return self.module.isNameDefined(name) - else: - return False - + defs = chain(defs, [self._localNameToFullName_map[name]]) + + if before is not None: + def _f(o: Documentable | ImportAlias) -> bool: + return o.linenumber < before + defs = filter(_f, defs) + + try: + i0 = next(iter(defs)) + except StopIteration: + if not isinstance(self, Module): + return self.module.getDefinitions(name) + # not found + return [] + + return sorted(chain([i0], defs), key=attrgetter('linenumber')) + class Module(CanContainImportsDocumentable): kind = DocumentableKind.MODULE @@ -503,15 +516,6 @@ def setup(self) -> None: self._docformat: Optional[str] = None - def _localNameToFullName(self, name: str) -> str: - if name in self.contents: - o: Documentable = self.contents[name] - return o.fullName() - elif name in self._localNameToFullName_map: - return self._localNameToFullName_map[name] - else: - return name - @property def module(self) -> 'Module': return self @@ -545,7 +549,7 @@ class Package(Module): kind = DocumentableKind.PACKAGE # List of exceptions class names in the standard library, Python 3.8.10 -_STD_LIB_EXCEPTIONS = ('ArithmeticError', 'AssertionError', 'AttributeError', +_STD_LIB_EXCEPTIONS = set(('ArithmeticError', 'AssertionError', 'AttributeError', 'BaseException', 'BlockingIOError', 'BrokenPipeError', 'BufferError', 'BytesWarning', 'ChildProcessError', 'ConnectionAbortedError', 'ConnectionError', @@ -565,7 +569,7 @@ class Package(Module): 'SystemExit', 'TabError', 'TimeoutError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', - 'ValueError', 'Warning', 'ZeroDivisionError') + 'ValueError', 'Warning', 'ZeroDivisionError')) def is_exception(cls: 'Class') -> bool: """ Whether is class should be considered as @@ -573,6 +577,12 @@ def is_exception(cls: 'Class') -> bool: kind L{DocumentableKind.EXCEPTION}. """ for base in cls.mro(True, False): + if not isinstance(base, str): + base = base.fullName() + if base.startswith('builtins.'): + base = base[9:] + if '.' in base: + continue if base in _STD_LIB_EXCEPTIONS: return True return False @@ -804,15 +814,6 @@ def find(self, name: str) -> Optional[Documentable]: return obj return None - def _localNameToFullName(self, name: str) -> str: - if name in self.contents: - o: Documentable = self.contents[name] - return o.fullName() - elif name in self._localNameToFullName_map: - return self._localNameToFullName_map[name] - else: - return self.parent._localNameToFullName(name) - @property def constructor_params(self) -> Mapping[str, Optional[ast.expr]]: """A mapping of constructor parameter names to their type annotation. @@ -842,12 +843,12 @@ def docsources(self) -> Iterator[Documentable]: for b in self.parent.mro(include_self=False): if self.name in b.contents: yield b.contents[self.name] - - def _localNameToFullName(self, name: str) -> str: - return self.parent._localNameToFullName(name) - def isNameDefined(self, name: str) -> bool: - return self.parent.isNameDefined(name) + def isNameDefined(self, name: str, before:int|None=None) -> bool: + return self.parent.isNameDefined(name, before) + + def getDefinitions(self, name: str, before:int|None=None) -> list[Documentable | ImportAlias]: + return self.parent.getDefinitions(name, before) class Function(Inheritable): kind = DocumentableKind.FUNCTION @@ -873,6 +874,14 @@ class FunctionOverload: signature: Signature decorators: Sequence[ast.expr] +@object.__new__ +class _NotYetResolved: + def __bool__(self) -> bool: + return False + +if TYPE_CHECKING: + _NotYetResolvedT = Type[_NotYetResolved] + class Attribute(Inheritable): kind: Optional[DocumentableKind] = DocumentableKind.ATTRIBUTE annotation: Optional[ast.expr] = None @@ -884,6 +893,24 @@ class Attribute(Inheritable): None value means the value is not initialized at the current point of the the process. """ + alias: Optional[str] = None + """" + We store the alias value here so we don't have to process it all the time. + + For aliases, this is the same as:: + + '.'.join(node2dottedname(self.value)) + + For other attributes, it's C{None}. + """ + + resolved_alias: _NotYetResolvedT | Documentable | None = _NotYetResolved + """ + Once we have resolved the alias to a Documentable object in post-processing, it's value is stored in this attribute. + + If it's None, it means that the alias could not be resolved to an object in the system. + """ + # Work around the attributes of the same name within the System class. _ModuleT = Module _PackageT = Package @@ -1512,6 +1539,24 @@ def defaultPostProcess(system:'System') -> None: for attrib in system.objectsOfType(Attribute): _inherits_instance_variable_kind(attrib) + _resolve_alias(attrib) + +def _resolve_alias(attr: Attribute) -> None: + if attr.kind is not DocumentableKind.ALIAS: + return + # Since we try to resolve all aliases once in post-processing, + # we use some caching + from pydoctor import names + resolved = names._expandAlias(attr.parent, attr) + if resolved: + resolved_ob = attr.system.objForFullName(resolved) + if resolved_ob: + attr.resolved_alias = resolved_ob + if attr not in resolved_ob.aliases: + resolved_ob.aliases.append(attr) + else: + # Not in the system + attr.resolved_alias = None def _inherits_instance_variable_kind(attr: Attribute) -> None: """ diff --git a/pydoctor/names.py b/pydoctor/names.py new file mode 100644 index 000000000..1c618b4c6 --- /dev/null +++ b/pydoctor/names.py @@ -0,0 +1,247 @@ +""" +Module containing the logic to resolve names, aliases and imports. +""" +from typing import Optional, List, TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Protocol +else: + Protocol = object + +from pydoctor import model + +class _IndirectionT(Protocol): + # This protocol is implemented by model.Attribute and model.ImportAlias + system: model.System + name: str + parent: model.CanContainImportsDocumentable + linenumber : int + alias: Optional[str] + def fullName(self) -> str:... + +def _localDocumentableToFullName(ctx: model.CanContainImportsDocumentable, o: 'model.Documentable', indirections:Optional[List['_IndirectionT']]) -> str: + """ + If the documentable is an alias, then follow it and return the supposed full name fo the documentable object, + or return the passed object's - C{o} - full name. + + Calls L{_resolveAlias} if the documentable is an alias. + """ + if o.kind is model.DocumentableKind.ALIAS: + assert isinstance(o, model.Attribute) + return _expandAlias(ctx, o, indirections) + return o.fullName() + +def _localNameToFullName(ctx: model.Documentable, name: str, indirections:Optional[List['_IndirectionT']]) -> str: + if isinstance(ctx, model.CanContainImportsDocumentable): + # Local names and aliases + if name in ctx.contents: + return _localDocumentableToFullName(ctx, ctx.contents[name], indirections) + + # Imports + if name in ctx._localNameToFullName_map: + return _expandImport(ctx, ctx._localNameToFullName_map[name], indirections) + + # Not found + if isinstance(ctx, model.Class): + # for classes, we try the upper scope. + return _localNameToFullName(ctx.parent, name, indirections) + else: + raise LookupError() + else: + assert ctx.parent is not None + return _localNameToFullName(ctx.parent, name, indirections) + +_ensure_indirection_list = lambda indirections: indirections if isinstance(indirections, list) else [] + +def _expandImport(ctx: model.CanContainImportsDocumentable, + import_: _IndirectionT, + indirections:Optional[List['_IndirectionT']]) -> str: + indirections = _ensure_indirection_list(indirections) + [import_] + + failed = _failManyAlises(ctx, import_, indirections) + if failed: + return failed + + fullName = import_.alias + assert fullName is not None, f"Bad import: {ctx.module.description}:{import_.linenumber}" + + allobjects = ctx.system.allobjects + + # the imported name is part of the system + if fullName in allobjects: + # the imported name might be an alias, so use _localDocumentableToFullName + resolved = _localDocumentableToFullName(ctx, allobjects[fullName], indirections) + if resolved: + return resolved + + dottedName = fullName.split('.') + parentName, targetName = '.'.join(dottedName[0:-1]), dottedName[-1] + + # the imported name is not part of the system, but it's parent is, + # so try to resolve the name from the parent's context. + # this logic has a blind spot: i the parent of the imported name is not found but + # the grand-parent exists in the system, it will not be used to resolve the imports "chain". + # We clould use a while loop to walk grand parents until there are no more. + if parentName in allobjects: + parent = allobjects[parentName] + try: + return _localNameToFullName(parent, targetName, indirections) + except LookupError: + # The name does not exists or the module is not completely processed yet. + pass + + return fullName + +def _failManyAlises(ctx: model.CanContainImportsDocumentable, alias: _IndirectionT, indirections:Optional[List[_IndirectionT]]=None) -> Optional[str]: + """ + Returns None if the alias can be resolved normally, + returns a string and log a warning if the alias + can't be resolved because it's too complex. + """ + if indirections and len(indirections) > ctx._RESOLVE_ALIAS_MAX_RECURSE: + ctx.module.report("Too many aliases", lineno_offset=alias.linenumber, section='aliases') + return indirections[0].fullName() + return None + +# TODO: This function should be applicable for imports sa well. +# or maybe part of this function should also be applicable. Some special +# care needs to be taken while resolving an ALIAS vs an IMPORT because an alias +# can have the same name and target and redirect to the upper scope name, +# so this needs to be handled specially. +def _expandAlias(self: model.CanContainImportsDocumentable, + alias: _IndirectionT, + indirections:Optional[List[_IndirectionT]]=None) -> str: + """ + Resolve the indirection value to it's target full name. + Or fall back to original name if we've exhausted the max recursions (or something else went wrong). + + @param alias: an indirection (alias or import) + @param indirections: Chain of indirection objects followed. + This variable is used to prevent infinite loops when doing the lookup. + @returns: The potential full name of the + """ + indirections = _ensure_indirection_list(indirections) + + failed = _failManyAlises(self, alias, indirections) + if failed: + return failed + + # the alias attribute should never be None for indirections objects + name = alias.alias + assert name, f"Bad alias: {self.module.description}:{alias.linenumber}" + + # the context is important + ctx = self + + if alias not in indirections: + # We redirect to the original object + return ctx.expandName(name, indirections=indirections + [alias]) + + # We try the upper scope only if we detect a direct cycle. + # Otherwise just fail. + if alias is not indirections[-1]: + self.module.report("Can't resolve cyclic aliases", lineno_offset=alias.linenumber, section='aliases') + return indirections[0].fullName() + + # Issue tracing the alias back to it's original location, found the same alias again. + parent = ctx.parent + if parent is not None and not isinstance(self, model.Module): + # We try with the parent scope. + # This is used in situations like right here in the System class and it's aliases (before version > 22.5.1), + # because they have the same name as the name they are aliasing, the alias resolves to the same object. + # We could use astuce here to be more precise (better static analysis) and make this code more simple and less error-prone. + return parent.expandName(name, indirections=indirections + [alias]) + + self.module.report("Failed to resolve alias (found same alias again)", lineno_offset=alias.linenumber, section='aliases') + return indirections[0].fullName() + +def expandName(self:model.Documentable, name: str, indirections:Optional[List[_IndirectionT]]=None) -> str: + """ + Return a fully qualified name for the possibly-dotted `name`. + + To explain what this means, consider the following modules: + + mod1.py:: + + from external_location import External + class Local: + pass + + mod2.py:: + + from mod1 import External as RenamedExternal + import mod1 as renamed_mod + class E: + pass + + In the context of mod2.E, C{expandName("RenamedExternal")} should be + C{"external_location.External"} and C{expandName("renamed_mod.Local")} + should be C{"mod1.Local"}. + + This method is in charge to follow the aliases when possible! + It will reccursively follow any L{DocumentableKind.ALIAS} entry found + up to certain level of complexity. + + Example: + + mod1.py:: + + import external + class Processor: + spec = external.Processor.more_spec + P = Processor + + mod2.py:: + + from mod1 import P + class Runner: + processor = P + + In the context of mod2, C{expandName("Runner.processor.spec")} should be + C{"external.Processor.more_spec"}. + + @param name: The name to expand. + @param indirections: See L{_resolveAlias} + @note: The implementation replies on iterating through the each part of the dotted name, + calling L{_localNameToFullName} for each name in their associated context and incrementally building + the fullName from that. + + Lookup members in superclasses when possible and follows L{DocumentableKind.ALIAS}. This mean that L{expandName} will never return the name of an alias, + it will always follow it's indirection to the origin. + """ + + parts = name.split('.') + ctx: model.Documentable = self # The context for the currently processed part of the name. + for i, part in enumerate(parts): + if i > 0 and not isinstance(ctx, model.CanContainImportsDocumentable): + # Stop now, this is a big blind spot of this function. + # The problem is that trying to resolve an attribute (because i > 0 meaning we're resolving an attribute part of the name) + # within the context of another attribute or function will always fallback to the parent scope, which is simply wrong. + # So we stop resolving the name when we encounter something that is not a class or module. + full_name = f'{ctx.fullName()}.{part}' + break + try: + full_name = _localNameToFullName(ctx, part, indirections) + except LookupError: + if i == 0: + full_name = part + else: + full_name = None + # The local name was not found. + # If we're looking at a class, we try our luck with the inherited members + if isinstance(ctx, model.Class): + inherited = ctx.find(part) + if inherited: + full_name = inherited.fullName() + if full_name is None: + # We don't have a full name + # TODO find a way to indicate if the expanded name is "guessed" or if we have the the correct fullName. + # With the current implementation, this would mean checking if "parts[i + 1:]" contains anything. + full_name = f'{ctx.fullName()}.{part}' + break + nxt = self.system.objForFullName(full_name) + if nxt is None: + break + ctx = nxt + + return '.'.join([full_name] + parts[i + 1:]) diff --git a/pydoctor/sphinx.py b/pydoctor/sphinx.py index dbec697a3..cadc5d152 100644 --- a/pydoctor/sphinx.py +++ b/pydoctor/sphinx.py @@ -135,6 +135,12 @@ def getLink(self, name: str) -> Optional[str]: """ Return link for `name` or None if no link is found. """ + # special casing the 'builtins' module because our name resolving + # replaces bare builtins names with builtins. in order not to confuse + # them with objects in the system when reparenting. + if name.startswith('builtins.'): + name = name[9:] + base_url, relative_link = self._links.get(name, (None, None)) if not relative_link: return None diff --git a/pydoctor/templatewriter/pages/__init__.py b/pydoctor/templatewriter/pages/__init__.py index 68e013ec0..30cd1f6ce 100644 --- a/pydoctor/templatewriter/pages/__init__.py +++ b/pydoctor/templatewriter/pages/__init__.py @@ -4,7 +4,8 @@ from typing import ( TYPE_CHECKING, Dict, Iterator, List, Optional, Mapping, Sequence, Type, Union -) +) + import ast import abc @@ -21,7 +22,7 @@ from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval if TYPE_CHECKING: - from typing_extensions import Final + from typing import Final from twisted.web.template import Flattenable from pydoctor.templatewriter.pages.attributechild import AttributeChild from pydoctor.templatewriter.pages.functionchild import FunctionChild @@ -333,7 +334,8 @@ def objectExtras(self, ob: model.Documentable) -> List["Flattenable"]: epydoc2stan.safe_to_stan(extra, ob.docstring_linker, ob, fallback = lambda _,__,___:epydoc2stan.BROKEN, section='extra'))) return r - + # Not adding Known aliases here because it would really be too much information. + # TODO: Would it actully be TMI? def functionBody(self, ob: model.Documentable) -> "Flattenable": return self.docgetter.get(ob) @@ -370,6 +372,13 @@ class ModulePage(CommonPage): def extras(self) -> List["Flattenable"]: r: List["Flattenable"] = [] + # Add Known aliases, for modules. + aliases = sorted(self.ob.aliases, key=util.alphabetical_order_func) + p = assembleList(self.ob.system, "Known aliases: ", + [o.fullName() for o in aliases], self.page_url) + if p is not None: + r.append(tags.p(p)) + sourceHref = util.srclink(self.ob) if sourceHref: r.append(tags.a("(source)", href=sourceHref, class_="sourceLink")) @@ -418,20 +427,16 @@ def assembleList( lst = lst2 if not lst: return None - def one(item: str) -> "Flattenable": + r: List['Flattenable'] = [] + for i, item in enumerate(lst): + if i>0: + r.append(', ') if item in system.allobjects: - return tags.code(epydoc2stan.taglink(system.allobjects[item], page_url)) + r.append(tags.code(epydoc2stan.taglink(system.allobjects[item], page_url))) else: - return item - def commasep(items: Sequence[str]) -> List["Flattenable"]: - r = [] - for item in items: - r.append(one(item)) - r.append(', ') - del r[-1] - return r + r.append(tags.code(item)) p: List["Flattenable"] = [label] - p.extend(commasep(lst)) + p.extend(r) return p @@ -468,6 +473,14 @@ def extras(self) -> List["Flattenable"]: [o.fullName() for o in subclasses], self.page_url) if p is not None: r.append(tags.p(p)) + + # Add Known aliases, for classes. TODO: move this to extra_info + aliases = sorted(self.ob.aliases, key=util.alphabetical_order_func) + if aliases: + p = assembleList(self.ob.system, "Known aliases: ", + [o.fullName() for o in aliases], self.page_url) + if p is not None: + r.append(tags.p(p)) constructor = epydoc2stan.get_constructors_extra(self.ob) if constructor: diff --git a/pydoctor/templatewriter/pages/attributechild.py b/pydoctor/templatewriter/pages/attributechild.py index e22e84fc2..a5f2604cf 100644 --- a/pydoctor/templatewriter/pages/attributechild.py +++ b/pydoctor/templatewriter/pages/attributechild.py @@ -5,7 +5,7 @@ from twisted.web.iweb import ITemplateLoader from twisted.web.template import Tag, renderer, tags -from pydoctor.model import Attribute +from pydoctor.model import Attribute, DocumentableKind from pydoctor import epydoc2stan from pydoctor.templatewriter import TemplateElement, util from pydoctor.templatewriter.pages import format_decorators @@ -78,7 +78,14 @@ def functionBody(self, request: object, tag: Tag) -> "Flattenable": @renderer def constantValue(self, request: object, tag: Tag) -> "Flattenable": - if self.ob.kind not in self.ob.system.show_attr_value or self.ob.value is None: - return tag.clear() - # Attribute is a constant/type alias (with a value), then display it's value - return epydoc2stan.format_constant_value(self.ob) + if self.ob.value is not None: + if self.ob.kind in self.ob.system.show_attr_value: + # Attribute is a constant/type alias (with a value), then display it's value + return epydoc2stan.format_constant_value(self.ob) + if self.ob.kind is DocumentableKind.ALIAS: + # Attribute is an alias, use special formatting. + return epydoc2stan.format_alias_value(self.ob) + else: + return '' + else: + return '' diff --git a/pydoctor/templatewriter/summary.py b/pydoctor/templatewriter/summary.py index 36bd5adea..84d3b1ec6 100644 --- a/pydoctor/templatewriter/summary.py +++ b/pydoctor/templatewriter/summary.py @@ -1,12 +1,12 @@ """Classes that generate the summary pages.""" from __future__ import annotations -from collections import defaultdict from typing import ( TYPE_CHECKING, DefaultDict, Dict, Iterable, List, Mapping, MutableSet, Sequence, Tuple, Type, Union, cast ) +from collections import defaultdict from twisted.web.template import Element, Tag, TagLoader, renderer, tags from pydoctor import epydoc2stan, model, linker diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index 053342be4..8ab2381aa 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -2,16 +2,18 @@ import ast import sys -from pydoctor import astbuilder, astutils, model -from pydoctor import epydoc2stan +from pydoctor import astbuilder, astutils, model, names, epydoc2stan from pydoctor.epydoc.markup import DocstringLinker, ParsedDocstring from pydoctor.options import Options from pydoctor.stanutils import flatten, html2stan, flatten_text from pydoctor.epydoc.markup.epytext import Element, ParsedEpytextDocstring -from pydoctor.epydoc2stan import format_summary, get_parsed_type +from pydoctor.epydoc2stan import ensure_parsed_docstring, format_summary, get_parsed_type from pydoctor.test.test_packages import processPackage + from pydoctor.utils import partialclass +from pydoctor.extensions import ExtRegistrar, ModuleVisitorExt + from . import CapSys, NotFoundLinker, posonlyargs, typecomment import pytest @@ -473,6 +475,7 @@ class A: ''' src_c = ''' from b import B as C + # We don't support implicit re-exports for now ''' src_d = ''' from c import C @@ -490,7 +493,12 @@ class D(C): assert isinstance(D, model.Class) # An older version of this test expected a.A as the result. # Read the comment in test_aliasing() to learn why this was changed. - assert D.bases == ['c.C'] + # --- + # Changed again in 2022, we require the object to be re-exported with __all__ + # At some point, we might support implicit re-exports as well, in which case + # this test wil fail again with value ['c.C'] ! + + assert D.bases == ['a.A'] @systemcls_param def test_aliasing_recursion(systemcls: Type[model.System]) -> None: @@ -508,12 +516,12 @@ class D(C): assert D.bases == ['mod.C'], D.bases @systemcls_param -def test_documented_no_alias(systemcls: Type[model.System]) -> None: - """A variable that is documented should not be considered an alias.""" - # TODO: We should also verify this for inline docstrings, but the code - # currently doesn't support that. We should perhaps store aliases - # as Documentables as well, so we can change their 'kind' when - # an inline docstring follows the assignment. +def test_documented_alias(systemcls: Type[model.System]) -> None: + """ + All variables that simply points to an attribute or name are now + legit L{Attribute} documentable objects with a special kind: L{DocumentableKind.ALIAS}. + """ + mod = fromText(''' class SimpleClient: pass @@ -522,14 +530,551 @@ class Processor: @ivar clientFactory: Callable that returns a client. """ clientFactory = SimpleClient - ''', systemcls=systemcls) + ''', systemcls=systemcls, modname='mod') P = mod.contents['Processor'] f = P.contents['clientFactory'] assert unwrap(f.parsed_docstring) == """Callable that returns a client.""" assert f.privacyClass is model.PrivacyClass.PUBLIC - assert f.kind is model.DocumentableKind.INSTANCE_VARIABLE + # we now mark aliases with the ALIAS kind! + assert f.kind is model.DocumentableKind.ALIAS assert f.linenumber + # Verify this is working with inline docstrings as well., + # but the code + # currently doesn't support that. We should perhaps store aliases + # as Documentables as well, so we can change their 'kind' when + # an inline docstring follows the assignment. + mod = fromText(''' + class SimpleClient: + pass + class Processor: + clientFactory = SimpleClient + """ + Callable that returns a client. + """ + ''', systemcls=systemcls, modname='mod') + P = mod.contents['Processor'] + f = P.contents['clientFactory'] + ensure_parsed_docstring(f) + assert unwrap(f.parsed_docstring) == """Callable that returns a client.""" + assert f.privacyClass is model.PrivacyClass.VISIBLE + # we now mark aliases with the ALIAS kind! + assert f.kind is model.DocumentableKind.ALIAS + assert f.linenumber + + +@systemcls_param +def test_expandName_alias(systemcls: Type[model.System]) -> None: + """ + expandName now follows all kinds of aliases! + """ + system = systemcls() + fromText(''' + class BaseClient: + BAR = 1 + FOO = 2 + ''', system=system, modname='base_mod') + mod = fromText(''' + import base_mod as _base + class SimpleClient(_base.BaseClient): + BARS = SimpleClient.FOO + FOOS = _base.BaseClient.BAR + class Processor: + var = 1 + clientFactory = SimpleClient + BARS = _base.BaseClient.FOO + P = Processor + ''', system=system, modname='mod') + Processor = mod.contents['Processor'] + assert mod.expandName('Processor.clientFactory')=="mod.SimpleClient" + assert mod.expandName('Processor.BARS')=="base_mod.BaseClient.FOO" + assert mod.system.allobjects.get("mod.SimpleClient") is not None + assert mod.system.allobjects.get("mod.SimpleClient.FOO") is None + assert mod.contents['P'].kind is model.DocumentableKind.ALIAS + + from pydoctor import names + + assert names._expandAlias(mod, cast(model.Attribute, mod.contents['P']), None)=="mod.Processor" + assert names._localNameToFullName(mod, 'P', None)=="mod.Processor" + + assert mod.expandName('P')=="mod.Processor" + assert mod.expandName('P.var')=="mod.Processor.var" + assert mod.expandName('P.clientFactory')=="mod.SimpleClient" + assert mod.expandName('Processor.clientFactory.BARS')=="base_mod.BaseClient.FOO" + assert mod.expandName('Processor.clientFactory.FOOS')=="base_mod.BaseClient.BAR" + assert mod.expandName('P.clientFactory.BARS')=="base_mod.BaseClient.FOO" + assert mod.expandName('P.clientFactory.FOOS')=="base_mod.BaseClient.BAR" + assert Processor.expandName('clientFactory')=="mod.SimpleClient" + assert Processor.expandName('BARS')=="base_mod.BaseClient.FOO" + assert Processor.expandName('clientFactory.BARS')=="base_mod.BaseClient.FOO" + +@systemcls_param +def test_expandName_alias_same_name_recursion(systemcls: Type[model.System]) -> None: + """ + When the name of the alias is the same as the name contained in it's value, + it can create a recursion error. The C{indirections} parameter of methods + L{CanContainImportsDocumentable._localNameToFullName}, L{Documentable._resolveAlias} and L{Documentable.expandName} prevent an infinite loop where + the name it beening revolved to the object itself. When this happends, we use the parent object context + to call L{Documentable.expandName()}, avoiding the infinite recursion. + """ + system = systemcls() + base_mod = fromText(''' + class Foo: + _1=1 + _2=2 + _3=3 + foo = Foo._1 + class Attribute: + foo = foo + class Class: + pass + ''', system=system, modname='base_mod') + mod = fromText(''' + from base_mod import Attribute, Class, Foo + class System: + Attribute = Attribute + Class = Class + class SuperSystem: + foo = Foo._3 + class Attribute: + foo = foo + Attribute = Attribute + ''', system=system, modname='mod') + System = mod.contents['System'] + SuperSystem = mod.contents['SuperSystem'] + assert mod.expandName('System.Attribute')=="base_mod.Attribute" + assert mod.expandName('System.Class')=="base_mod.Class" + + assert System.expandName('Attribute')=="base_mod.Attribute" + assert System.expandName('Class')=="base_mod.Class" + + assert mod.expandName('SuperSystem.Attribute')=="mod.SuperSystem.Attribute" + assert SuperSystem.expandName('Attribute')=="mod.SuperSystem.Attribute" + + assert mod.expandName('SuperSystem.Attribute.foo')=="base_mod.Foo._3" + assert SuperSystem.expandName('Attribute.foo')=="base_mod.Foo._3" + + assert base_mod.contents['Attribute'].contents['foo'].kind is model.DocumentableKind.ALIAS + assert mod.contents['System'].contents['Attribute'].kind is model.DocumentableKind.ALIAS + + assert base_mod.contents['Attribute'].contents['foo'].fullName() == 'base_mod.Attribute.foo' + assert 'base_mod.Attribute.foo' in mod.system.allobjects, str(list(mod.system.allobjects)) + + f = mod.system.objForFullName('base_mod.Attribute.foo') + assert isinstance(f, model.Attribute) + assert f.kind is model.DocumentableKind.ALIAS + + assert mod.expandName('System.Attribute.foo')=="base_mod.Foo._1" + assert System.expandName('Attribute.foo')=="base_mod.Foo._1" + + assert mod.contents['System'].contents['Attribute'].kind is model.DocumentableKind.ALIAS + + # Tests the .aliases property. + + assert [o.fullName() for o in base_mod.contents['Foo'].contents['_1'].aliases] == ['base_mod.foo','base_mod.Attribute.foo'] + assert [o.fullName() for o in base_mod.contents['Foo'].contents['_3'].aliases] == ['mod.SuperSystem.foo', 'mod.SuperSystem.Attribute.foo'] + +@systemcls_param +def test_expandName_import_alias_wins_over_module_level_alias(systemcls: Type[model.System]) -> None: + """ + + """ + system = systemcls() + fromText(''' + ssl = 1 + ''', system=system, modname='twisted.internet') + mod = fromText(''' + try: + from twisted.internet import ssl as _ssl + except ImportError: + # this code is currently simply signored + _ssl = None + ''', system=system, modname='mod') + + assert mod.expandName('_ssl')=="twisted.internet.ssl" + s = mod.resolveName('_ssl') + assert isinstance(s, model.Attribute) + assert s.value is not None + assert ast.literal_eval(s.value)==1 + +@systemcls_param +def test_expandName_alias_documentable_module_level(systemcls: Type[model.System]) -> None: + + system = systemcls() + fromText(''' + ssl = 1 + ''', system=system, modname='twisted.internet') + mod = fromText(''' + try: + from twisted.internet import ssl as _ssl + # This will create a Documentable entry + ssl = _ssl + except ImportError: + # this code is ignored + ssl = None + ''', system=system, modname='mod') + + assert mod.expandName('ssl')=="twisted.internet.ssl" + assert mod.expandName('_ssl')=="twisted.internet.ssl" + s = mod.resolveName('ssl') + assert isinstance(s, model.Attribute) + assert s.value is not None + assert ast.literal_eval(s.value)==1 + assert mod.contents['ssl'].kind is model.DocumentableKind.ALIAS + +@systemcls_param +def test_expandName_alias_not_documentable_class_level(systemcls: Type[model.System], capsys: CapSys) -> None: + """ + """ + system = systemcls() + mod = fromText(''' + import sys + class A: + if sys.version_info[0] > 3: + alias = B.b + else: + # this code is ignored + alias = B.a + class B: + a = 3 + b = 4 + ''', system=system, modname='mod') + + A = mod.contents['A'] + # assert capsys.readouterr().out == '', A.contents['alias'] + s = mod.resolveName('A.alias') + assert isinstance(s, model.Attribute) + assert s.fullName() == "mod.B.b", (names._localNameToFullName(A, 'alias', None), A.contents['alias']) + assert s.value is not None + assert ast.literal_eval(s.value)==4 + assert mod.contents['A'].contents['alias'].kind is model.DocumentableKind.ALIAS + +@systemcls_param +def test_expandName_alias_documentale_class_level(systemcls: Type[model.System]) -> None: + system = systemcls() + mod = fromText(''' + import sys + class A: + alias = None + if sys.version_info[0] > 3: + alias = B.b + else: + # this code is currently simply signored + # because it's not in a 'body'. + alias = B.a + class B: + a = 3 + b = 4 + ''', system=system, modname='mod') + + s = mod.resolveName('A.alias') + assert isinstance(s, model.Attribute) + assert s.fullName() == "mod.B.b" + assert s.value is not None + assert ast.literal_eval(s.value)==4 + assert mod.contents['A'].contents['alias'].kind is model.DocumentableKind.ALIAS + +@systemcls_param +def test_aliases_property(systemcls: Type[model.System]) -> None: + base_mod = ''' + class Z: + pass + ''' + src = ''' + import base_mod + from abc import ABC + class A(ABC): + _1=1 + _2=2 + _3=3 + class_ = B # this is a forward reference + + class B(A): + _1=a_1 + _2=A._2 + _3=A._3 + class_ = a + + a = A + a_1 = A._1 + b = B + bob = b.class_.class_.class_ + lol = b.class_.class_ + blu = b.class_ + mod = base_mod + ''' + system = systemcls() + fromText(base_mod, system=system, modname='base_mod') + fromText(src, system=system) + + A = system.allobjects['.A'] + B = system.allobjects['.B'] + _base_mod = system.allobjects['base_mod'] + + assert isinstance(A, model.Class) + assert A.subclasses == [system.allobjects['.B']] + + assert [o.fullName() for o in A.aliases] == ['.B.class_', '.a', '.bob', '.blu'] + assert [o.fullName() for o in B.aliases] == ['.A.class_', '.b', '.lol'] + assert [o.fullName() for o in A.contents['_1'].aliases] == ['.B._1', '.a_1'] + assert [o.fullName() for o in A.contents['_2'].aliases] == ['.B._2'] + assert [o.fullName() for o in A.contents['_3'].aliases] == ['.B._3'] + assert [o.fullName() for o in _base_mod.aliases] == ['.mod'] + + # Aliases cannot currently have aliases because resolveName() always follows the aliases. + assert [o.fullName() for o in A.contents['class_'].aliases] == [] + assert [o.fullName() for o in B.contents['class_'].aliases] == [] + +@systemcls_param +def test_aliases_re_export(systemcls: Type[model.System]) -> None: + + src = ''' + # Import and re-export some external lib + + from constantly import NamedConstant, ValueConstant, FlagConstant, Names, Values, Flags + from mylib import core + from mylib.core import Observable + from mylib.core._impl import Processor + Patator = core.Patator + + __all__ = ["NamedConstant", "ValueConstant", "FlagConstant", "Names", "Values", "Flags", + "Processor","Patator","Observable"] + ''' + system = systemcls() + fromText(src, system=system) + assert system.allobjects['.ValueConstant'].kind is model.DocumentableKind.ALIAS + n = system.allobjects['.NamedConstant'] + assert isinstance(n, model.Attribute) + assert astutils.unparse(n.value).strip() == 'constantly.NamedConstant' == astutils.node2fullname(n.value, n.parent) + + n = system.allobjects['.Processor'] + assert isinstance(n, model.Attribute) + assert n.kind is model.DocumentableKind.ALIAS + assert astutils.unparse(n.value).strip() == 'mylib.core._impl.Processor' == astutils.node2fullname(n.value, n.parent) + + assert system.allobjects['.ValueConstant'].kind is model.DocumentableKind.ALIAS + n = system.allobjects['.Observable'] + assert isinstance(n, model.Attribute) + assert n.kind is model.DocumentableKind.ALIAS + assert astutils.unparse(n.value).strip() == 'mylib.core.Observable' == astutils.node2fullname(n.value, n.parent) + + n = system.allobjects['.Patator'] + assert isinstance(n, model.Attribute) + assert n.kind is model.DocumentableKind.ALIAS + assert astutils.unparse(n.value).strip() == 'core.Patator' + assert astutils.node2fullname(n.value, n.parent) == 'mylib.core.Patator' + +@systemcls_param +def test_exportName_re_exported_aliases(systemcls: Type[model.System]) -> None: + """ + What if we re-export an alias? + """ + + base_mod = ''' + class Zoo: + _1=1 + class Hey: + _2=2 + Z = Zoo + H = Hey + ''' + src = ''' + from base_mod import Z, H + __all__ = ["Z", "H"] + ''' + system = systemcls() + + builder = system.systemBuilder(system) + builder.addModuleString(base_mod, modname='base_mod') + builder.addModuleString(src, modname='mod') + builder.buildModules() + + mod = system.allobjects['mod'] + bmod = system.allobjects['base_mod'] + alias = system.allobjects['mod.Z'] + + assert mod.expandName('Z') == "base_mod.Zoo" # Should be "base_mod.Zoo" + assert mod.expandName('Z._1') == "base_mod.Zoo._1" # Should be "base_mod.Zoo._1", linked to https://github.com/twisted/pydoctor/issues/295 + + assert bmod.expandName('Z._1') == "base_mod.Zoo._1" + assert bmod.expandName('Zoo._1') == "base_mod.Zoo._1" + + assert isinstance(alias, model.Attribute) + assert alias.kind is model.DocumentableKind.ALIAS + assert alias.alias == 'base_mod.Zoo' + + # This should not be None! + assert alias.resolved_alias is not None + + +@systemcls_param +def test_expandName_aliasloops(systemcls: Type[model.System]) -> None: + + src = ''' + from abc import ABC + class A(ABC): + _1=C._2 + _2=2 + + class B(A): + _1=A._2 + _2=A._1 + + class C(A,B): + _1=A._1 + _2=B._2 + # this could crash with an infitine recursion error! + ''' + system = systemcls() + fromText(src, system=system) + A = system.allobjects['.A'] + B = system.allobjects['.B'] + C = system.allobjects['.C'] + + assert A.expandName('_1') == '.A._1' + assert B.expandName('_2') == '.B._2' + assert C.expandName('_2') == '.C._2' + assert C.expandName('_1') == '.C._1' + +@systemcls_param +def test_import_name_already_defined(systemcls: Type[model.System]) -> None: + # from cpython asyncio/__init__.py + mod1src = """ + def get_running_loop(): + ... + + def set_running_loop(loop): + ... + + try: + from _asyncio import (get_running_loop, set_running_loop,) + except ImportError: + pass + """ + + mod2src = """ + # For pydoctor, this will import the mod1 version if the names, + # This is probably an implementation limitation. + # We don't handle duplicates and ambiguity in the best manner. + # The imports are stored in a different dict than the documentable, + # It's checked after the contents entries, which could be overriden by + # a name in the _localNameToFullName_map, but we don't check that currently. + # It only works when explicitely re-exported with __all__. + + from mod1 import * + """ + + system = systemcls() + builder = system.systemBuilder(system) + builder.addModuleString(mod1src, 'mod1', is_package=True) + builder.addModuleString(mod2src, 'mod2', is_package=True) + builder.buildModules() + + mod1 = system.allobjects['mod1'] + mod2 = system.allobjects['mod2'] + + assert mod2.expandName('get_running_loop') == 'mod1.get_running_loop' + assert mod2.expandName('set_running_loop') == 'mod1.set_running_loop' + + assert list(mod1.contents) == ['get_running_loop', 'set_running_loop'] + +@systemcls_param +def test_re_export_name_defined(systemcls: Type[model.System], capsys: CapSys) -> None: + # from cpython asyncio/__init__.py + mod1src = """ + + # this will export the _asyncio version if the names instead! + __all__ = ( + 'set_running_loop', + 'get_running_loop', + ) + + def get_running_loop(): + ... + + def set_running_loop(loop): + ... + + try: + from _asyncio import (get_running_loop, set_running_loop,) + except ImportError: + pass + """ + + mod2src = """ + # for pydoctor, this will import the _asyncio version if the names instead! + from mod1 import * + """ + + system = systemcls() + builder = system.systemBuilder(system) + builder.addModuleString(mod1src, 'mod1', is_package=True) + builder.addModuleString(mod2src, 'mod2', is_package=True) + builder.buildModules() + + mod1 = system.allobjects['mod1'] + mod2 = system.allobjects['mod2'] + + assert mod2.expandName('get_running_loop') == '_asyncio.get_running_loop' + assert mod2.expandName('set_running_loop') == '_asyncio.set_running_loop' + + assert mod1.contents['get_running_loop'].kind == model.DocumentableKind.ALIAS + assert mod1.contents['set_running_loop'].kind == model.DocumentableKind.ALIAS + + assert list(mod1.contents) == ['get_running_loop', 'set_running_loop'] + + out = capsys.readouterr().out + assert all(s in out for s in ["duplicate Function 'mod1.get_running_loop'", "duplicate Function 'mod1.set_running_loop'"]), out + +@systemcls_param +def test_re_export_name_defined_alt(systemcls: Type[model.System], capsys: CapSys) -> None: + # from cpython asyncio/__init__.py + mod1src= """ + + # this will export the local version if the names, + # because they are defined after the imports of the same names. + __all__ = ( + 'set_running_loop', + 'get_running_loop', + ) + + err = False + try: + from _asyncio import (get_running_loop, set_running_loop,) + except ImportError: + err = True + + if err: + def get_running_loop(): + ... + + def set_running_loop(loop): + ... + """ + + mod2src = """ + # for pydoctor, this will import the mod1 version if the names. + # Even if in the test code, the most correct thing to do would be + # to export the _asyncio. + # Since pydoctor does not proceed with a path-sentive AST analysis, + # the names defined in the "if err:" overrides the imports, even if + # in therory we could determine their exclusivity and choose the best one. + + from mod1 import * + """ + + system = systemcls() + builder = system.systemBuilder(system) + builder.addModuleString(mod1src, 'mod1', is_package=True) + builder.addModuleString(mod2src, 'mod2', is_package=True) + builder.buildModules() + + mod2 = system.allobjects['mod2'] + + assert mod2.expandName('get_running_loop') == 'mod1.get_running_loop' + assert mod2.expandName('set_running_loop') == 'mod1.set_running_loop' + + @systemcls_param def test_subclasses(systemcls: Type[model.System]) -> None: src = ''' @@ -2062,6 +2607,199 @@ class j: pass assert all(n in system.allobjects['top'].contents for n in ['f', 'g', 'h', 'i', 'j']) @systemcls_param +def test_module_level_attributes_and_aliases(systemcls: Type[model.System]) -> None: + """ + Currently, the first analyzed assigment wins, basically. I believe further logic should be added + such that definitions in the orelse clause of the Try node is processed before the + except handlers. This way could define our aliases both there and in the body of the + Try node and fall back to what's defnied in the handlers if the names doesn't exist yet. + """ + system = systemcls() + builder = system.systemBuilder(system) + builder.addModuleString(''' + ssl = 1 + ''', modname='twisted.internet') + builder.addModuleString(''' + try: + from twisted.internet import ssl as _ssl + # The first analyzed assigment to an alias wins. + ssl = _ssl + # For classic variables, the rules are the same. + var = 1 + # For constants, the rules are still the same. + VAR = 1 + # Looks like a constant, but should be treated like an alias + ALIAS = _ssl + except ImportError: + ssl = None + var = 2 + VAR = 2 + ALIAS = None + ''', modname='mod') + builder.buildModules() + mod = system.allobjects['mod'] + + # Test alias + assert mod.expandName('ssl')=="twisted.internet.ssl" + assert mod.expandName('_ssl')=="twisted.internet.ssl" + s = mod.resolveName('ssl') + assert isinstance(s, model.Attribute) + assert s.value is not None + assert ast.literal_eval(s.value)==1 + assert s.kind == model.DocumentableKind.VARIABLE + + # Test variable + assert mod.expandName('var')=="mod.var" + v = mod.resolveName('var') + assert isinstance(v, model.Attribute) + assert v.value is not None + assert ast.literal_eval(v.value)==1 + assert v.kind == model.DocumentableKind.VARIABLE + + # Test look like constant + assert mod.expandName('VAR')=="mod.VAR" + V = mod.resolveName('VAR') + assert isinstance(V, model.Attribute) + assert V.value is not None + assert ast.literal_eval(V.value)==1 + assert V.kind == model.DocumentableKind.VARIABLE + + # Test looks like constant but actually an alias. + assert mod.expandName('ALIAS')=="twisted.internet.ssl" + s = mod.resolveName('ALIAS') + assert isinstance(s, model.Attribute) + assert s.value is not None + assert ast.literal_eval(s.value)==1 + assert s.kind == model.DocumentableKind.VARIABLE + +@systemcls_param +def test_alias_instance_method_same_name(systemcls: Type[model.System], capsys: CapSys) -> None: + code = ''' + class Log: + def fatal(self, msg, *args): + ... + + _global_log = Log() + fatal = _global_log.fatal + ''' + mod = fromText(code, systemcls=systemcls) + # assert not capsys.readouterr().out + capsys.readouterr() + + fatal = mod.contents['fatal'] + global_log = mod.contents['_global_log'] + assert isinstance(fatal, model.Attribute) + + assert names._expandAlias(mod, fatal) == '._global_log.fatal' + assert names.expandName(fatal, '_global_log.fatal') == '._global_log.fatal' + assert not capsys.readouterr().out + + # this is NOT an issue: + assert global_log.expandName('fatal') == '._global_log.fatal' + +@pytest.mark.xfail +@systemcls_param +def test_links_function_docstring_imports_in_body(systemcls: Type[model.System]) -> None: + + mod = ''' + def expandName(name: str) -> str: + """ + See L{names.expandName} + """ + from pydoctor import names + return names.expandName(name) + ''' + + _impl = ''' + def expandName(name: str) -> str: + """Docs""" + ... + ''' + + system = systemcls() + builder = system.systemBuilder(system) + builder.addModuleString('', modname='pydoctor', is_package=True) + builder.addModuleString(mod, modname='pydoctor.model') + builder.addModuleString(_impl, modname='pydoctor.names') + builder.buildModules() + + fn = system.allobjects['pydoctor.model.expandName'] + + assert fn.expandName('names.expandName') == 'pydoctor.names.expandName' + +@systemcls_param +def test_import_aliases_across_modules(systemcls: Type[model.System]) -> None: + """ + We should be able to follow import aliases across several modules. + """ + system = systemcls() + builder = system.systemBuilder(system) + + builder.addModuleString(''' + from _impl2 import i as _i, j + from ._impl import f as _f + # __all__ not defined, so nothing get re-exported here + # but we should be able to follow the aliases anyhow. + ''', modname='top', is_package=True) + + builder.addModuleString(''' + def f(): + pass + ''', modname='_impl', parent_name='top') + + builder.addModuleString(''' + class i: pass + class j: pass + ''', modname='_impl2') + + builder.addModuleString(''' + from top import j,_i,_f + ''', modname='client') + + builder.buildModules() + + client = system.allobjects['client'] + top = system.allobjects['top'] + _impl = system.allobjects['top._impl'] + assert isinstance(client, model.Module) + assert isinstance(top, model.Module) + assert client._localNameToFullName_map['_f'].alias == 'top._f' + assert top._localNameToFullName_map['_f'].alias == 'top._impl.f' + assert _impl.contents['f'] + + assert system.allobjects['client'].expandName('_f') == 'top._impl.f' + assert system.allobjects['client'].expandName('_i') == '_impl2.i' + assert system.allobjects['client'].expandName('j') == '_impl2.j' + + assert system.allobjects['client'].resolveName('_f') == system.allobjects['top._impl'].contents['f'] + assert system.allobjects['client'].resolveName('_i') == system.allobjects['_impl2'].contents['i'] + assert system.allobjects['client'].resolveName('j') == system.allobjects['_impl2'].contents['j'] + +@systemcls_param +def test_import_aliases_across_modules_cycle(systemcls: Type[model.System]) -> None: + + system = systemcls() + builder = system.systemBuilder(system) + + builder.addModuleString(''' + from _impl3 import _impl1 + ''', modname='_impl') + + builder.addModuleString(''' + from _impl import _impl1 + ''', modname='_impl2') + + builder.addModuleString(''' + from _impl import _impl1 + ''', modname='_impl3') + + builder.buildModules() + + assert system.allobjects['_impl3'].expandName('_impl1') == '_impl3._impl1' + assert system.allobjects['_impl2'].expandName('_impl1') == '_impl2._impl1' + assert system.allobjects['_impl'].expandName('_impl1') == '_impl._impl1' + # TODO: test the warnings +@systemcls_param def test_exception_kind(systemcls: Type[model.System], capsys: CapSys) -> None: """ Exceptions are marked with the special kind "EXCEPTION". @@ -2742,6 +3480,117 @@ def test_typealias_unstring(systemcls: Type[model.System]) -> None: # there is not Constant nodes in the type alias anymore next(n for n in ast.walk(typealias.value) if isinstance(n, ast.Constant)) +@systemcls_param +def test_expand_name_cycle_proof_while_visiting(systemcls: Type[model.System]) -> None: + src_inteface = '''\ + from zope.interface import Interface + from top.impl import Address + + class IAddress(Interface): + ... + ''' + + src_impl = '''\ + from zope.interface import implementer + from top.iface import IAddress + + @implementer(IAddress) + class Address(object): + ... + ''' + system = systemcls() + + class myExt(ModuleVisitorExt): + def depart_ClassDef(self, cls: ast.ClassDef): + ctx = self.visitor.builder.current + assert isinstance(ctx, model.Module) + + if ctx.name == 'iface': + assert ctx.expandName('Address') == 'top.impl.Address' + if ctx.name == 'impl': + + assert list(ctx._localNameToFullName_map) == ['implementer', 'IAddress'] + assert ctx._localNameToFullName_map['IAddress'].alias == 'top.iface.IAddress' + assert ctx.expandName('IAddress') == 'top.iface.IAddress' + + ExtRegistrar(system).register_astbuilder_visitor(myExt) + + builder = system.systemBuilder(system) + builder.addModuleString('', 'top', is_package=True) + builder.addModuleString(src_inteface, 'iface', parent_name='top') + builder.addModuleString(src_impl, 'impl', parent_name='top') + builder.buildModules() + +@systemcls_param +def test_builtins_aliases(systemcls: Type[model.System], capsys:CapSys) -> None: + src = ''' + """ + @var unicode: The type of Unicode strings, C{unicode} on Python 2 and C{str} + on Python 3. + """ + from io import StringIO + unicode = str + set = set + class list(list): + ... + xrange = range + StringIO = StringIO + myList = list + __all__ = [ + 'unicode', 'set', 'xrange', 'StringIO', 'list', + ] + ''' + + src2 = ''' + """ + Ref to L{unicode}. + """ + ''' + + system = systemcls() + # system.options.verbosity = 6 + builder = system.systemBuilder(system) + builder.addModuleString('', 'twisted', is_package=True) + builder.addModuleString('', 'python', parent_name='twisted', is_package=True) + builder.addModuleString(src, 'compat', parent_name='twisted.python') + builder.addModuleString('', 'pack', parent_name='twisted', is_package=True) + builder.addModuleString(src2, modname='thing', parent_name='twisted.pack') + builder.buildModules() + compat = system.allobjects['twisted.python.compat'] + strio = system.allobjects['twisted.python.compat.StringIO'] + setal = system.allobjects['twisted.python.compat.set'] + listal = system.allobjects['twisted.python.compat.myList'] + assert isinstance(strio, model.Attribute) + assert list(compat._localNameToFullName_map) == ['StringIO'] + assert len(strio.getDefinitions('StringIO', before=10)) == 1 + assert len(strio.getDefinitions('StringIO')) == 2 + assert strio.alias == 'io.StringIO' + assert setal.alias == 'builtins.set' + assert listal.alias == 'twisted.python.compat.list' + assert list(compat.contents)==['unicode', 'StringIO', 'set', 'list', 'xrange', 'myList'] + + thing = system.allobjects['twisted.pack.thing'] + thing.docstring_linker.link_xref('unicode', 'unicode', 3) + + assert capsys.readouterr().out == '' + +# These two tests should go with linker's tests +# TODO: Test the scenario of twisted JID: meaning a docstring links to JID, but JID is not defined in the current +# module, instead it's defined in a uncle module, but other uncle module does from x import y as JID +# The linker should ngive the priority to the declaration of JID instead of the import +# If the declaration doesn't exist then it uses the import. + +# TODO: Test the scenario of twisted URL: meaning it's re-exported from an external library at two different places +# a docstring link to URL but it's defined in modules as well (both re-export the alias but one of them is used only and +# we do not complain) + +# TODO: Test that an instance variable is never flagged as an alias. +# class F: +# def __init__(self, a): +# self.a = a +# class G(F): +# a = Exception # a is not an alias + @systemcls_param def test_mutilple_docstrings_warnings(systemcls: Type[model.System], capsys: CapSys) -> None: """ @@ -2792,4 +3641,5 @@ class B2: ''' fromText(src, systemcls=systemcls) # TODO: handle doc comments.x - assert capsys.readouterr().out == ':18: Existing docstring at line 14 is overriden\n' \ No newline at end of file + assert capsys.readouterr().out == ':18: Existing docstring at line 14 is overriden\n' + diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index b35a57062..d404fc51d 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -1176,14 +1176,13 @@ def test_EpydocLinker_resolve_identifier_xref_intersphinx_link_not_found(capsys: # This is called for the L{ext_module} markup. assert sut.link_to('ext_module', 'ext').tagName == '' assert not capsys.readouterr().out + sut._resolve_identifier_xref('ext_module', 0) with raises(LookupError): - sut._resolve_identifier_xref('ext_module', 0) + sut._resolve_identifier_xref('notfound', 0) captured = capsys.readouterr().out expected = ( - 'ignore-name:???: Cannot find link target for "ext_package.ext_module", ' - 'resolved from "ext_module" ' - '(you can link to external docs with --intersphinx)\n' + 'ignore-name:???: Cannot find link target for "notfound"\n' ) assert expected == captured diff --git a/pydoctor/test/test_mro.py b/pydoctor/test/test_mro.py index 2c96dc1a3..d82820d4d 100644 --- a/pydoctor/test/test_mro.py +++ b/pydoctor/test/test_mro.py @@ -58,7 +58,7 @@ class GenericPedalo(MyGeneric[ast.AST], Pedalo):... assert_mro_equals(mod.contents["D1"], ['mro.D1', 'mro.B1', 'mro.C1', 'mro.A1']) assert_mro_equals(mod.contents["E1"], ['mro.E1', 'mro.C1', 'mro.B1', 'mro.A1']) assert_mro_equals(mod.contents["Extension"], ["mro.Extension", "mod.External"]) - assert_mro_equals(mod.contents["MycustomString"], ["mro.MycustomString", "str"]) + assert_mro_equals(mod.contents["MycustomString"], ["mro.MycustomString", "builtins.str"]) assert_mro_equals( mod.contents["PedalWheelBoat"], diff --git a/pydoctor/test/test_packages.py b/pydoctor/test/test_packages.py index fe16a9991..46a52435b 100644 --- a/pydoctor/test/test_packages.py +++ b/pydoctor/test/test_packages.py @@ -2,7 +2,7 @@ from typing import Callable import pytest -from pydoctor import model +from pydoctor import model, names testpackages = Path(__file__).parent / 'testpackages' @@ -138,25 +138,21 @@ def test_reparenting_follows_aliases() -> None: assert isinstance(mything, model.Module) assert isinstance(myotherthing, model.Module) - assert mything._localNameToFullName('MyClass') == 'reparenting_follows_aliases.main.MyClass' - assert myotherthing._localNameToFullName('MyClass') == 'reparenting_follows_aliases._mything.MyClass' + assert names._localNameToFullName(mything, 'MyClass', None) == 'reparenting_follows_aliases.main.MyClass' + # This resolves to the re-exported full-Name of the class. + assert names._localNameToFullName(myotherthing, 'MyClass', None) == 'reparenting_follows_aliases.main.MyClass' system.find_object('reparenting_follows_aliases._mything.MyClass') == klass - # This part of the test cannot pass for now since we don't recursively resolve aliases. + # We now recursively resolve aliases. # See https://github.com/twisted/pydoctor/pull/414 and https://github.com/twisted/pydoctor/issues/430 - try: - assert system.find_object('reparenting_follows_aliases._myotherthing.MyClass') == klass - assert myotherthing.resolveName('MyClass') == klass - assert mything.resolveName('MyClass') == klass - assert top.resolveName('_myotherthing.MyClass') == klass - assert top.resolveName('_mything.MyClass') == klass - except (AssertionError, LookupError): - return - else: - raise AssertionError("Congratulation!") - + assert system.find_object('reparenting_follows_aliases._myotherthing.MyClass') == klass + assert myotherthing.resolveName('MyClass') == klass + assert mything.resolveName('MyClass') == klass + assert top.resolveName('_myotherthing.MyClass') == klass + assert top.resolveName('_mything.MyClass') == klass + @pytest.mark.parametrize('modname', ['reparenting_crash','reparenting_crash_alt']) def test_reparenting_crash(modname: str) -> None: """ diff --git a/pydoctor/test/test_zopeinterface.py b/pydoctor/test/test_zopeinterface.py index 9c9654286..a520cfd53 100644 --- a/pydoctor/test/test_zopeinterface.py +++ b/pydoctor/test/test_zopeinterface.py @@ -219,6 +219,7 @@ class IMyInterface(interface.Interface): ''' mod = fromText(src, systemcls=systemcls) attr = mod.contents['IMyInterface'].contents['attribute'] + assert mod.contents['IMyInterface'].contents['Attrib'].kind is model.DocumentableKind.ALIAS assert attr.docstring == 'fun in a bun' assert attr.kind is model.DocumentableKind.ATTRIBUTE diff --git a/tox.ini b/tox.ini index d4352576c..36efd41a0 100644 --- a/tox.ini +++ b/tox.ini @@ -88,6 +88,25 @@ commands = {toxworkdir}/cpython/Lib | tee {toxworkdir}/cpython-output/run.log" pytest -vv docs/tests/test_standard_library_docs.py +[testenv:cpython-intersphinx] +description = Parse CPython API and build intersphinx only (fastest cpython test) +deps = + pytest +commands = + sh -c "if [ ! -d {toxworkdir}/cpython ]; then \ + git clone --depth 1 https://github.com/python/cpython.git {toxworkdir}/cpython; \ + fi" + sh -c "cd {toxworkdir}/cpython && git pull" + touch {toxworkdir}/cpython/Lib/__init__.py + rm -rf {toxworkdir}/cpython-intersphinx-out + pydoctor \ + --docformat=plaintext \ + --project-base-dir={toxworkdir}/cpython \ + --html-output={toxworkdir}/cpython-intersphinx-out \ + --make-intersphinx \ + {toxworkdir}/cpython/Lib + + [testenv:numpy-apidocs] description = Build numpy API documentation. For now we don't check for any warnings or other errors. The only purpose of this test is to make sure pydoctor doesn't crash. deps =