From bf5716aba0879272704fe0cdd5e71fdb0bb7d61b Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sat, 17 Sep 2022 12:35:43 -0400 Subject: [PATCH 01/48] Add test --- pydoctor/test/test_astbuilder.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index c48462f33..1db285953 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -2115,3 +2115,17 @@ def test_prepend_package_real_path(systemcls: Type[model.System]) -> None: finally: systemcls.systemBuilder = _builderT_init +@systemcls_param +def test_constructor_signature_init_and_new(systemcls: Type[model.System]) -> None: + """ + Pydoctor can infer the constructor signature when both __new__ and __init__ are + """ + + src1 = '''\ + class Animal(object): + def __new__(cls, name): + print('__new__() called.') + obj = super().__new__(cls) + obj.name = name + return obj + ''' From f7d0d1a8cfe6233cbe9eceeb4dc4a78284422b28 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sat, 17 Sep 2022 12:50:25 -0400 Subject: [PATCH 02/48] Add src --- pydoctor/test/test_astbuilder.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index 1db285953..10cd2fc6b 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -2124,8 +2124,20 @@ def test_constructor_signature_init_and_new(systemcls: Type[model.System]) -> No src1 = '''\ class Animal(object): def __new__(cls, name): - print('__new__() called.') - obj = super().__new__(cls) - obj.name = name - return obj + print('__new__() called.') + obj = super().__new__(cls) + obj.name = name # not recognized by pydoctor + return obj ''' + + src2 = '''\ + class Animal(object): + # Can be omitted, Python will give a default implementation + def __new__(cls, *args, **kw): + print('__new__() called.') + print('args: ', args, ', kw: ', kw) + return super().__new__(cls) + + def __init__(self, name): + print('__init__() called.') + self.name = name''' From d87e7459ed6505c85b67f29a00d3cfda3ea470e0 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 18 Sep 2022 17:26:33 -0400 Subject: [PATCH 03/48] Structure tests --- pydoctor/test/test_astbuilder.py | 98 ++++++++++++++++++++++++++++---- 1 file changed, 88 insertions(+), 10 deletions(-) diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index 10cd2fc6b..699efe038 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -2116,23 +2116,53 @@ def test_prepend_package_real_path(systemcls: Type[model.System]) -> None: systemcls.systemBuilder = _builderT_init @systemcls_param -def test_constructor_signature_init_and_new(systemcls: Type[model.System]) -> None: - """ - Pydoctor can infer the constructor signature when both __new__ and __init__ are - """ +def test_constructor_signature_init(systemcls: Type[model.System]) -> None: + + src = '''\ + class Person(object): + # pydoctor can infer the constructor to be: "Person(name, age)" + def __init__(self, name, age): + self.name = name + self.age = age + + class Citizen(Person): + # pydoctor can infer the constructor to be: "Citizen(nationality, *args, **kwargs)" + def __init__(self, nationality, *args, **kwargs): + self.nationality = nationality + super(Citizen, self).__init__(*args, **kwargs) + ''' + + # assert that Person.extra_info contains the available constructors names + # Like "Available constructor: ``Person(name, age)``" that links to Person.__init__ documentation. + + # assert that Citizen.extra_info contains the available constructors names + # Like "Available constructor: ``Citizen(nationality, *args, **kwargs)``" that links to Citizen.__init__ documentation. - src1 = '''\ +@systemcls_param +def test_constructor_signature_new(systemcls: Type[model.System]) -> None: + src = '''\ class Animal(object): + # pydoctor can infer the constructor to be: "Animal(name)" def __new__(cls, name): - print('__new__() called.') obj = super().__new__(cls) - obj.name = name # not recognized by pydoctor + # assignation not recognized by pydoctor, attribute 'name' will not be documented + obj.name = name return obj ''' - src2 = '''\ +@systemcls_param +def test_constructor_signature_init_and_new(systemcls: Type[model.System]) -> None: + """ + Pydoctor can't infer the constructor signature when both __new__ and __init__ are defined. + __new__ takes the precedence over __init__ because it's called first. Trying to infer what are the complete + constructor signature when __new__ is defined might be very hard because the method can return an instance of + another class, calling another __init__ method. We're not there yet in term of static analysis. + """ + + src = '''\ class Animal(object): - # Can be omitted, Python will give a default implementation + # both __init__ and __new__ are defined, pydoctor only looks at the __new__ method + # pydoctor infers the constructor to be: "Animal(*args, **kw)" def __new__(cls, *args, **kw): print('__new__() called.') print('args: ', args, ', kw: ', kw) @@ -2140,4 +2170,52 @@ def __new__(cls, *args, **kw): def __init__(self, name): print('__init__() called.') - self.name = name''' + self.name = name + + class Cat(Animal): + # Idem, but __new__ is inherited. + # pydoctor infers the constructor to be: "Cat(*args, **kw)" + # This is why it's important to still document __init__ as a regular method. + def __init__(self, name, owner): + super().__init__(name) + self.owner = owner + ''' + +@systemcls_param +def test_constructor_signature_classmethod(systemcls: Type[model.System]) -> None: + + src = '''\ + + def get_default_options() -> 'Options': + """ + This is another constructor for class 'Options'. + But it's not recognized by pydoctor at this moment. + """ + return Options() + + class Options: + a,b,c = None, None, None + + @classmethod + def create_no_hints(cls): + """ + Pydoctor can't deduce that this method is a constructor as well, + because there is no type annotation. + """ + return cls() + + # thanks to type hints, + # pydoctor can infer the constructor to be: "Options.create()" + @staticmethod + def create() -> 'Options': + # the fictional constructor is not detected by pydoctor, because it doesn't exists actually. + return Options(1,2,3) + + # thanks to type hints, + # pydoctor can infer the constructor to be: "Options.create_from_num(num)" + @classmethod + def create_from_num(cls, num) -> 'Options': + c = cls.create() + c.a = num + return c + ''' From f022cdf585b41cc2f6ba6885b59495af5d5f1fdd Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 18 Sep 2022 17:27:16 -0400 Subject: [PATCH 04/48] wip Class.constructors attribute --- pydoctor/model.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pydoctor/model.py b/pydoctor/model.py index fdbcd7c07..a423b6852 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -600,6 +600,7 @@ def setup(self) -> None: self.rawbases: Sequence[Tuple[str, ast.expr]] = [] self.raw_decorators: Sequence[ast.expr] = [] self.subclasses: List[Class] = [] + self.constructors: List[Function] = [] self._initialbases: List[str] = [] self._initialbaseobjects: List[Optional['Class']] = [] @@ -612,6 +613,37 @@ def _init_mro(self) -> None: except ValueError as e: self.report(str(e), 'mro') self._mro = list(self.allbases(True)) + + def _init_constructors(self) -> None: + """ + Initiate the L{Class.constructors} list. + """ + # If __new__ is defined, then it takes precedence over __init__ + _new = self.find('__new__') + if isinstance(_new, Function): + self.constructors.append(_new) + elif _new is None: + _init = self.find('__init__') + if isinstance(_init, Function): + self.constructors.append(_init) + + # Then look for staticmethod/classmethod constructors + for fun in self.contents.values(): + if not isinstance(fun, Function): + continue + + if not fun.kind in (DocumentableKind.STATIC_METHOD, DocumentableKind.CLASS_METHOD): + continue + + # get return annotation, if it returns the same type as self, it's a constructor method. + if not 'return' in fun.annotations: + continue + + # pydoctor only understand explicit annotation + # it does not comprehend the Self-type for instance. + return_ann = astutils.node2fullname(fun.annotations['return'], self) + if return_ann == self.fullName(): + self.constructors.append(fun) @overload def mro(self, include_external:'Literal[True]', include_self:bool=True) -> List[Union['Class', str]]:... From d578d4c9e6d28286c95cb0960e536f1de6f5064d Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Mon, 19 Sep 2022 12:42:15 -0400 Subject: [PATCH 05/48] Add extra information with constructor signature that links to the detected constructor Function. --- pydoctor/epydoc2stan.py | 57 ++++++++++++++++++++++++++++ pydoctor/model.py | 45 ++++++++++++++++------ pydoctor/test/test_astbuilder.py | 48 +++++++++++++++++++++-- pydoctor/test/test_templatewriter.py | 13 +++++++ 4 files changed, 147 insertions(+), 16 deletions(-) diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index f7a3dcaac..6abfd02d4 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -980,3 +980,60 @@ def insert_break_points(text: str) -> 'Flattenable': r += [tags.wbr(), '.'] return tags.transparent(*r) +def format_constructor_short_text(constructor: model.Constructor, forclass: model.Class) -> str: + """ + Returns a simplified signature of the constructor. + """ + args = '' + for index, (name, ann) in enumerate(constructor.annotations.items()): + if name=='return': + continue + + # Special casing __new__ because it's actually a static method + if index==0 and (constructor.name in ('__new__', '__init__') or + constructor.kind is model.DocumentableKind.CLASS_METHOD): + # Omit first argument (self/cls) from mini signature. + continue + star = '' + if isinstance(name, VariableArgument): + star='*' + elif isinstance(name, KeywordArgument): + star='**' + + if args: + args += ', ' + + args+= f"{star}{name}" + + # display innner classes with their name starting at the top level class. + _current = forclass + class_name = [] + while isinstance(_current, model.Class): + class_name.append(_current.name) + _current = _current.parent + + callable_name = '.'.join(reversed(class_name)) + + if constructor.name not in ('__new__', '__init__'): + # We assume that the constructor is a method accessible in the Class. + + callable_name += f'.{constructor.name}' + + return f"{callable_name}({args})" + +def populate_constructors_extra_info(cls:model.Class) -> None: + """ + Adds an extra information to be rendered based on Class constructors. + """ + from pydoctor.templatewriter import util + visibleConstructors = [c for c in cls.constructors if c.isVisible] + if visibleConstructors: + plural = 's' if len(visibleConstructors)>1 else '' + extra_epytext = f'Constructor{plural}: ' + for i, c in enumerate(sorted(visibleConstructors, key=util.objects_order)): + if i != 0: + extra_epytext += ', ' + short_text = format_constructor_short_text(c, cls) + extra_epytext += '`%s <%s>`' % (short_text, c.fullName()) + + cls.extra_info.append(parse_docstring(cls, extra_epytext, cls, 'restructuredtext', section='constructor extra')) diff --git a/pydoctor/model.py b/pydoctor/model.py index a423b6852..ac00480b2 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -31,11 +31,11 @@ from pydoctor.sphinx import CacheT, SphinxInventory if TYPE_CHECKING: - from typing_extensions import Literal + from typing_extensions import Literal, Protocol from pydoctor.astbuilder import ASTBuilder, DocumentableT else: Literal = {True: bool, False: bool} - ASTBuilder = object + ASTBuilder = Protocol = object # originally when I started to write pydoctor I had this idea of a big @@ -585,11 +585,26 @@ def getbases(o:Union['Class', str]) -> List[Union['Class', str]]: init_finalbaseobjects(cls) return mro.mro(cls, getbases) +class Constructor(Protocol): + """ + Protocol implemented by L{Function} objects. + + Makes the assumption that the constructor name is available in the locals of the class + it's supposed to create. Typically with __init__ and __new__ it's always the case. But it also means that + no regular function (not classmethod or staticmethod) can be interpreted as a constructor for a given class. + """ + name: str + annotations: Mapping[str, Optional[ast.expr]] + kind: DocumentableKind + parent: CanContainImportsDocumentable + isVisible: bool + def fullName(self) -> str:... + class Class(CanContainImportsDocumentable): kind = DocumentableKind.CLASS parent: CanContainImportsDocumentable decorators: Sequence[Tuple[str, Optional[Sequence[ast.expr]]]] - + # set in post-processing: _finalbaseobjects: Optional[List[Optional['Class']]] = None _finalbases: Optional[List[str]] = None @@ -600,7 +615,7 @@ def setup(self) -> None: self.rawbases: Sequence[Tuple[str, ast.expr]] = [] self.raw_decorators: Sequence[ast.expr] = [] self.subclasses: List[Class] = [] - self.constructors: List[Function] = [] + self.constructors: List[Constructor] = [] self._initialbases: List[str] = [] self._initialbaseobjects: List[Optional['Class']] = [] @@ -616,9 +631,12 @@ def _init_mro(self) -> None: def _init_constructors(self) -> None: """ - Initiate the L{Class.constructors} list. + Initiate the L{Class.constructors} list. A constructor MUST be a method accessible + in the locals of the class. """ + # Look for python language powered constructors. # If __new__ is defined, then it takes precedence over __init__ + # Blind spot: we don't understand when a Class is using a metaclass that overrides __call__. _new = self.find('__new__') if isinstance(_new, Function): self.constructors.append(_new) @@ -627,23 +645,24 @@ def _init_constructors(self) -> None: if isinstance(_init, Function): self.constructors.append(_init) - # Then look for staticmethod/classmethod constructors + # Then look for staticmethod/classmethod constructors, + # This only happens at the local scope level (i.e not looking in super-classes). for fun in self.contents.values(): if not isinstance(fun, Function): continue - + # Only static methods and class methods can be recognized as constructors if not fun.kind in (DocumentableKind.STATIC_METHOD, DocumentableKind.CLASS_METHOD): continue - # get return annotation, if it returns the same type as self, it's a constructor method. if not 'return' in fun.annotations: continue - - # pydoctor only understand explicit annotation - # it does not comprehend the Self-type for instance. + # pydoctor understand explicit annotation as well as the Self-Type. return_ann = astutils.node2fullname(fun.annotations['return'], self) - if return_ann == self.fullName(): + if return_ann == self.fullName() or return_ann in ('typing.Self', 'typing_extensions.Self'): self.constructors.append(fun) + + from pydoctor import epydoc2stan + epydoc2stan.populate_constructors_extra_info(self) @overload def mro(self, include_external:'Literal[True]', include_self:bool=True) -> List[Union['Class', str]]:... @@ -1355,10 +1374,12 @@ def postProcess(self) -> None: # default post-processing includes: # - Processing of subclasses # - MRO computing. + # - Lookup of constructors # - Checking whether the class is an exception for cls in self.objectsOfType(Class): cls._init_mro() + cls._init_constructors() for b in cls.baseobjects: if b is not None: diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index 699efe038..deb57bee9 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -5,6 +5,7 @@ from pydoctor import astbuilder, astutils, model +from pydoctor import epydoc2stan from pydoctor.epydoc.markup import DocstringLinker, ParsedDocstring from pydoctor.options import Options from pydoctor.stanutils import flatten, html2stan, flatten_text @@ -2115,6 +2116,10 @@ def test_prepend_package_real_path(systemcls: Type[model.System]) -> None: finally: systemcls.systemBuilder = _builderT_init +def getConstructorsText(cls: model.Class) -> str: + return '\n'.join( + epydoc2stan.format_constructor_short_text(c, cls) for c in cls.constructors) + @systemcls_param def test_constructor_signature_init(systemcls: Type[model.System]) -> None: @@ -2131,12 +2136,13 @@ def __init__(self, nationality, *args, **kwargs): self.nationality = nationality super(Citizen, self).__init__(*args, **kwargs) ''' - - # assert that Person.extra_info contains the available constructors names + mod = fromText(src, systemcls=systemcls) + # Like "Available constructor: ``Person(name, age)``" that links to Person.__init__ documentation. + assert getConstructorsText(mod.contents['Person']) == "Person(name, age)" - # assert that Citizen.extra_info contains the available constructors names # Like "Available constructor: ``Citizen(nationality, *args, **kwargs)``" that links to Citizen.__init__ documentation. + assert getConstructorsText(mod.contents['Citizen']) == "Citizen(nationality, *args, **kwargs)" @systemcls_param def test_constructor_signature_new(systemcls: Type[model.System]) -> None: @@ -2150,6 +2156,10 @@ def __new__(cls, name): return obj ''' + mod = fromText(src, systemcls=systemcls) + + assert getConstructorsText(mod.contents['Animal']) == "Animal(name)" + @systemcls_param def test_constructor_signature_init_and_new(systemcls: Type[model.System]) -> None: """ @@ -2181,6 +2191,11 @@ def __init__(self, name, owner): self.owner = owner ''' + mod = fromText(src, systemcls=systemcls) + + assert getConstructorsText(mod.contents['Animal']) == "Animal(*args, **kw)" + assert getConstructorsText(mod.contents['Cat']) == "Cat(*args, **kw)" + @systemcls_param def test_constructor_signature_classmethod(systemcls: Type[model.System]) -> None: @@ -2189,7 +2204,7 @@ def test_constructor_signature_classmethod(systemcls: Type[model.System]) -> Non def get_default_options() -> 'Options': """ This is another constructor for class 'Options'. - But it's not recognized by pydoctor at this moment. + But it's not recognized by pydoctor because it's not defined in the locals of Options. """ return Options() @@ -2219,3 +2234,28 @@ def create_from_num(cls, num) -> 'Options': c.a = num return c ''' + + mod = fromText(src, systemcls=systemcls) + + assert getConstructorsText(mod.contents['Options']) == "Options.create()\nOptions.create_from_num(num)" + +@systemcls_param +def test_constructor_inner_class(systemcls: Type[model.System]) -> None: + src = '''\ + from typing import Self + class Animal(object): + class Bar(object): + # pydoctor can infer the constructor to be: "Animal.Bar(name)" + def __new__(cls, name): + ... + class Foo(object): + # pydoctor can infer the constructor to be: "Animal.Bar.Foo.create(name)" + @classmethod + def create(cls, name) -> 'Self': + c = cls.create() + c.a = num + return c + ''' + mod = fromText(src, systemcls=systemcls) + assert getConstructorsText(mod.contents['Animal'].contents['Bar']) == "Animal.Bar(name)" + assert getConstructorsText(mod.contents['Animal'].contents['Bar'].contents['Foo']) == "Animal.Bar.Foo.create(name)" diff --git a/pydoctor/test/test_templatewriter.py b/pydoctor/test/test_templatewriter.py index d5ef42373..4d142da34 100644 --- a/pydoctor/test/test_templatewriter.py +++ b/pydoctor/test/test_templatewriter.py @@ -737,3 +737,16 @@ def test_crash_xmlstring_entities_rst(capsys:CapSys, processtypes:bool) -> None: assert out == '\n'.join(warnings)+'\n' +def test_constructor_renders(capsys:CapSys) -> None: + ... + src = '''\ + class Animal(object): + # pydoctor can infer the constructor to be: "Animal(name)" + def __new__(cls, name): + ... + ''' + + mod = fromText(src) + html = getHTMLOf(mod.contents['Animal']) + assert 'Constructor: ' in html + assert 'Animal(name)' in html From 6f8e806b511ea5fb52a2fdef203206041714581c Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Mon, 19 Sep 2022 14:29:13 -0400 Subject: [PATCH 06/48] Introduce the SignatureBuilder class and refactor the astbuilder to use it. --- pydoctor/astbuilder.py | 125 ++++++++++++++++++++++++++++------------- 1 file changed, 85 insertions(+), 40 deletions(-) diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index 45e984cee..1a2cdce6e 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -4,14 +4,14 @@ import sys from functools import partial -from inspect import Parameter, Signature +from inspect import Parameter, Signature, _ParameterKind from itertools import chain from pathlib import Path from typing import ( Any, Callable, Collection, Dict, Iterable, Iterator, List, Mapping, Optional, Sequence, Tuple, - Type, TypeVar, Union, cast + Type, TypeVar, Union, cast, TYPE_CHECKING ) - +import attr import astor from pydoctor import epydoc2stan, model, node2stan, extensions from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval @@ -19,6 +19,12 @@ is__name__equals__main__, unstring_annotation, iterassign, NodeVisitor) + +if TYPE_CHECKING: + from typing import Protocol +else: + Protocol = object + def parseFile(path: Path) -> ast.Module: """Parse the contents of a Python source file.""" with open(path, 'rb') as f: @@ -810,46 +816,12 @@ def _handleFunctionDef(self, elif is_classmethod: func.kind = model.DocumentableKind.CLASS_METHOD - # Position-only arguments were introduced in Python 3.8. - posonlyargs: Sequence[ast.arg] = getattr(node.args, 'posonlyargs', ()) - - num_pos_args = len(posonlyargs) + len(node.args.args) - defaults = node.args.defaults - default_offset = num_pos_args - len(defaults) - def get_default(index: int) -> Optional[ast.expr]: - assert 0 <= index < num_pos_args, index - index -= default_offset - return None if index < 0 else defaults[index] - - parameters: List[Parameter] = [] - def add_arg(name: str, kind: Any, default: Optional[ast.expr]) -> None: - default_val = Parameter.empty if default is None else _ValueFormatter(default, ctx=func) - parameters.append(Parameter(name, kind, default=default_val)) - - for index, arg in enumerate(posonlyargs): - add_arg(arg.arg, Parameter.POSITIONAL_ONLY, get_default(index)) - - for index, arg in enumerate(node.args.args, start=len(posonlyargs)): - add_arg(arg.arg, Parameter.POSITIONAL_OR_KEYWORD, get_default(index)) - - vararg = node.args.vararg - if vararg is not None: - add_arg(vararg.arg, Parameter.VAR_POSITIONAL, None) - - assert len(node.args.kwonlyargs) == len(node.args.kw_defaults) - for arg, default in zip(node.args.kwonlyargs, node.args.kw_defaults): - add_arg(arg.arg, Parameter.KEYWORD_ONLY, default) - - kwarg = node.args.kwarg - if kwarg is not None: - add_arg(kwarg.arg, Parameter.VAR_KEYWORD, None) - try: - signature = Signature(parameters) + signature = signature_from_functiondef(node, func) except ValueError as ex: func.report(f'{func.fullName()} has invalid parameters: {ex}') signature = Signature() - + func.signature = signature func.annotations = self._annotations_from_function(node) @@ -933,7 +905,11 @@ def _get_all_ast_annotations() -> Iterator[Tuple[str, Optional[ast.expr]]]: name: None if value is None else unstring_annotation(value, self.builder.current) for name, value in _get_all_ast_annotations() } - + +class _ValueFormatterT(Protocol): + def __init__(self, value: Any, ctx: model.Documentable):... + def __repr__(self) -> str: ... + class _ValueFormatter: """ Class to encapsulate a python value and translate it to HTML when calling L{repr()} on the L{_ValueFormatter}. @@ -961,6 +937,75 @@ def __repr__(self) -> str: # but potential XML parser errors caused by XMLString needs to be handled later. return ''.join(node2stan.node2html(self._colorized.to_node(), self._linker)) +def signature_from_functiondef(node: Union[ast.AsyncFunctionDef, ast.FunctionDef], ctx: model.Documentable) -> Signature: + # Currently ignores type hints. + + # Position-only arguments were introduced in Python 3.8. + posonlyargs: Sequence[ast.arg] = getattr(node.args, 'posonlyargs', ()) + + num_pos_args = len(posonlyargs) + len(node.args.args) + defaults = node.args.defaults + default_offset = num_pos_args - len(defaults) + + def get_default(index: int) -> Optional[ast.expr]: + """ + Get the default value of the parameter, by index. + """ + assert 0 <= index < num_pos_args, index + index -= default_offset + return None if index < 0 else defaults[index] + + sigbuilder = SignatureBuilder(ctx=ctx) + + for index, arg in enumerate(posonlyargs): + sigbuilder.add_param(arg.arg, Parameter.POSITIONAL_ONLY, get_default(index)) + + for index, arg in enumerate(node.args.args, start=len(posonlyargs)): + sigbuilder.add_param(arg.arg, Parameter.POSITIONAL_OR_KEYWORD, get_default(index)) + + vararg = node.args.vararg + if vararg is not None: + sigbuilder.add_param(vararg.arg, Parameter.VAR_POSITIONAL) + + assert len(node.args.kwonlyargs) == len(node.args.kw_defaults) + for arg, default in zip(node.args.kwonlyargs, node.args.kw_defaults): + sigbuilder.add_param(arg.arg, Parameter.KEYWORD_ONLY, default) + + kwarg = node.args.kwarg + if kwarg is not None: + sigbuilder.add_param(kwarg.arg, Parameter.VAR_KEYWORD) + + return sigbuilder.get_signature() + +@attr.s(auto_attribs=True) +class SignatureBuilder: + """ + Builds a signature, parameter by parameter, with customizable value formatter and signature classes. + """ + ctx: model.Documentable + signature_class: Type['Signature'] = attr.ib(default=Signature) + value_formatter_class: Type['_ValueFormatterT'] = attr.ib(default=_ValueFormatter) + _parameters: List[Parameter] = attr.ib(factory=list, init=False) + _return_annotation: Any = attr.ib(default=Signature.empty, init=False) + + def add_param(self, name: str, + kind: _ParameterKind, + default: Optional[Any]=None, + annotation: Optional[Any]=None) -> None: + + default_val = Parameter.empty if default is None else self.value_formatter_class(default, self.ctx) + annotation_val = Parameter.empty if annotation is None else self.value_formatter_class(annotation, self.ctx) + self._parameters.append(Parameter(name, kind, default=default_val, annotation=annotation_val)) + + def set_return_annotation(self, annotation: Optional[Any]) -> None: + self._return_annotation = Signature.empty if annotation is None else self.value_formatter_class(annotation, self.ctx) + + def get_signature(self) -> Signature: + """ + @raises ValueError: If Signature() call fails. + """ + return self.signature_class(self._parameters, return_annotation=self._return_annotation) + def _infer_type(expr: ast.expr) -> Optional[ast.expr]: """Infer an expression's type. @param expr: The expression's AST. From 24d5ad62bda1abc9a1a8de3464ed00a41141ae6b Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Mon, 26 Sep 2022 20:15:37 -0400 Subject: [PATCH 07/48] wip --- pydoctor/astutils.py | 51 ++++++- pydoctor/extensions/attrs.py | 268 ++++++++++++++++++++++++++--------- pydoctor/test/test_attrs.py | 32 ++++- 3 files changed, 279 insertions(+), 72 deletions(-) diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index e2923e3cc..50b2aae30 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -4,14 +4,17 @@ import sys from numbers import Number -from typing import Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Union -from inspect import BoundArguments, Signature +from typing import Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Type, TypeVar, Union +from inspect import BoundArguments, Signature, Parameter import ast from pydoctor import visitor if TYPE_CHECKING: from pydoctor import model + from typing import Protocol +else: + Protocol = object # AST visitors @@ -84,6 +87,21 @@ def iterassign(node:_AssingT) -> Iterator[Optional[List[str]]]: dottedname = node2dottedname(target) yield dottedname +class _HasDecoratorList(Protocol): + decorator_list:List[ast.expr] + +def iter_decorators(node:_HasDecoratorList, ctx: 'model.Documentable') -> Iterator[Tuple[Optional[str], ast.AST]]: + """ + Utility function to iterate decorators. + """ + + for decnode in node.decorator_list: + namenode = decnode + if isinstance(namenode, ast.Call): + namenode = namenode.func + dottedname = node2fullname(namenode, ctx) + yield dottedname, decnode + def node2dottedname(node: Optional[ast.AST]) -> Optional[List[str]]: """ Resove expression composed by L{ast.Attribute} and L{ast.Name} nodes to a list of names. @@ -119,6 +137,35 @@ def bind_args(sig: Signature, call: ast.Call) -> BoundArguments: } return sig.bind(*call.args, **kwargs) +_T = TypeVar('_T') +def get_bound_literal(args:BoundArguments, name:str, typecheck:Type[_T]=object) -> Union[object, _T]: + """ + Retreive the literal value of an argument from the L{BoundArguments}. + Only works with purely literal values (no C{Name} or C{Attribute}). + + If the value is not present in the arguments, returns L{Parameter.empty}. + + @raises ValueError: If the passed value is not a literal or if it's not the right type. + """ + auto_attribs_expr = args.arguments.get(name) + if auto_attribs_expr is None: + return Parameter.empty + + try: + value = ast.literal_eval(auto_attribs_expr) + except ValueError: + message = ( + f'Unable to figure out value for {name!r} argument, maybe too complex', + ).replace("'", '"') + raise ValueError(message) + + if not isinstance(value, typecheck): + message = (f'Value for {name!r} argument ' + f'has type "{type(value).__name__}", expected {typecheck.__name__!r}', + ).replace("'", '"') + raise ValueError(message) + + return value if sys.version_info[:2] >= (3, 8): diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 887ddf7ac..5226c7e90 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -6,9 +6,9 @@ import ast import inspect -from typing import Optional, Union +from typing import Dict, Optional, Union -from pydoctor import astbuilder, model, astutils, extensions +from pydoctor import astbuilder, model, astutils, extensions, visitor import attr @@ -18,68 +18,39 @@ attrib_signature = inspect.signature(attr.ib) """Signature of the L{attr.ib} function for defining class attributes.""" -def uses_auto_attribs(call: ast.AST, module: model.Module) -> bool: - """Does the given L{attr.s()} decoration contain C{auto_attribs=True}? - @param call: AST of the call to L{attr.s()}. - This function will assume that L{attr.s()} is called without - verifying that. - @param module: Module that contains the call, used for error reporting. - @return: L{True} if L{True} is passed for C{auto_attribs}, - L{False} in all other cases: if C{auto_attribs} is not passed, - if an explicit L{False} is passed or if an error was reported. - """ +def get_attrs_args(call: ast.AST, ctx: model.Module) -> Optional[inspect.BoundArguments]: + """Get the arguments passed to an C{attr.s} class definition.""" if not isinstance(call, ast.Call): - return False - if not astutils.node2fullname(call.func, module) in ('attr.s', 'attr.attrs', 'attr.attributes'): - return False + return None try: - args = astutils.bind_args(attrs_decorator_signature, call) + return astutils.bind_args(attrs_decorator_signature, call) except TypeError as ex: message = str(ex).replace("'", '"') - module.report( + ctx.report( f"Invalid arguments for attr.s(): {message}", lineno_offset=call.lineno ) - return False - - auto_attribs_expr = args.arguments.get('auto_attribs') - if auto_attribs_expr is None: - return False - - try: - value = ast.literal_eval(auto_attribs_expr) - except ValueError: - module.report( - 'Unable to figure out value for "auto_attribs" argument ' - 'to attr.s(), maybe too complex', - lineno_offset=call.lineno - ) - return False - - if not isinstance(value, bool): - module.report( - f'Value for "auto_attribs" argument to attr.s() ' - f'has type "{type(value).__name__}", expected "bool"', - lineno_offset=call.lineno - ) - return False - - return value + return None def is_attrib(expr: Optional[ast.expr], ctx: model.Documentable) -> bool: """Does this expression return an C{attr.ib}?""" return isinstance(expr, ast.Call) and astutils.node2fullname(expr.func, ctx) in ( 'attr.ib', 'attr.attrib', 'attr.attr' ) + +def is_factory_call(expr: ast.expr, ctx: model.Documentable) -> bool: + """ + Does this AST represent a call to L{attr.Factory}? + """ + return isinstance(expr, ast.Call) and \ + astutils.node2fullname(expr.func, ctx) in ('attrs.Factory', 'attr.Factory') -def attrib_args(expr: ast.expr, ctx: model.Documentable) -> Optional[inspect.BoundArguments]: +def get_attrib_args(expr: ast.expr, ctx: model.Documentable) -> Optional[inspect.BoundArguments]: """Get the arguments passed to an C{attr.ib} definition. @return: The arguments, or L{None} if C{expr} does not look like an C{attr.ib} definition or the arguments passed to it are invalid. """ - if isinstance(expr, ast.Call) and astutils.node2fullname(expr.func, ctx) in ( - 'attr.ib', 'attr.attrib', 'attr.attr' - ): + if is_attrib(expr, ctx): try: return astutils.bind_args(attrib_signature, expr) except TypeError as ex: @@ -90,28 +61,102 @@ def attrib_args(expr: ast.expr, ctx: model.Documentable) -> Optional[inspect.Bou ) return None +def uses_init( + args:inspect.BoundArguments, + lineno: int, + ctx: model.Module, +) -> bool: + """ + Get the value of the C{init} argument passed to this L{attr.ib()}/L{attr.s()} call. + """ + try: + init = astutils.get_bound_literal(args, 'init', bool) + except ValueError as e: + ctx.report(str(e), lineno_offset=lineno) + return False + if init is inspect.Parameter.empty: + # default value for attr.ib(init) is True + return True + return init + +def uses_kw_only(args:inspect.BoundArguments, lineno:int, ctx: model.Module) -> bool: + """ + Get the value of the C{kw_only} argument passed to this L{attr.ib()}/L{attr.s()} call. + """ + try: + init = astutils.get_bound_literal(args, 'kw_only', bool) + except ValueError as e: + ctx.report(str(e), lineno_offset=lineno) + return False + if init is inspect.Parameter.empty: + # default value for attr.ib(kw_only) is True + return False + return init + +def uses_auto_attribs(args:inspect.BoundArguments, lineno:int, module: model.Module) -> bool: + """ + Get the value of the C{auto_attribs} argument passed to this L{attr.s()} call. + + @param call: AST of the call to L{attr.s()}. + This function will assume that L{attr.s()} is called without + verifying that. + @param module: Module that contains the call, used for error reporting. + @return: L{True} if L{True} is passed for C{auto_attribs}, + L{False} in all other cases: if C{auto_attribs} is not passed, + if an explicit L{False} is passed or if an error was reported. + """ + try: + value = astutils.get_bound_literal(args, 'auto_attribs', bool) + except ValueError as e: + module.report(str(e), lineno_offset=lineno) + return False + if value is inspect.Parameter.empty: + # default value is False for attr.s(auto_attribs) + return False + return value + def annotation_from_attrib( - self: astbuilder.ModuleVistor, - expr: ast.expr, - ctx: model.Documentable + args:inspect.BoundArguments, + ctx: model.Documentable, + for_init_method:bool=False ) -> Optional[ast.expr]: """Get the type of an C{attr.ib} definition. - @param expr: The L{ast.Call} expression's AST. + @param args: The L{inspect.BoundArguments} of the C{attr.ib()} call. @param ctx: The context in which this expression is evaluated. + @param for_init_method: Whether we're trying to figure out the __init__ parameter annotations + instead of the attribute annotations. @return: A type annotation, or None if the expression is not an C{attr.ib} definition or contains no type information. """ - args = attrib_args(expr, ctx) - if args is not None: - typ = args.arguments.get('type') - if typ is not None: - return astutils.unstring_annotation(typ, ctx) - default = args.arguments.get('default') - if default is not None: - return astbuilder._infer_type(default) + typ = args.arguments.get('type') + if typ is not None: + return astutils.unstring_annotation(typ, ctx) + default = args.arguments.get('default') + if default is not None: + return astbuilder._infer_type(default) + # TODO: support factory parameter. + if for_init_method: + # If a converter is defined, then we can't be sure of what exact type of parameter is accepted + converter = args.arguments.get('converter') + if converter is not None: + return ast.Constant(value=...) return None +def default_from_attrib(args:inspect.BoundArguments, ctx: model.Documentable) -> Optional[ast.AST]: + d = args.arguments.get('default') + f = args.arguments.get('factory') + if d is not None: + if is_factory_call(d, ctx): + return ast.Constant(value=...) + return d + elif f: # If a factory is defined, the default value is not obvious. + return ast.Constant(value=...) + else: + return None + class ModuleVisitor(extensions.ModuleVisitorExt): + + when = visitor.When.INNER def visit_ClassDef(self, node:ast.ClassDef) -> None: """ @@ -120,9 +165,45 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: cls = self.visitor.builder.current if not isinstance(cls, model.Class) or cls.name!=node.name: return - assert isinstance(cls, AttrsClass) - cls.auto_attribs = any(uses_auto_attribs(decnode, cls.module) for decnode in node.decorator_list) + + for name, decnode in astutils.iter_decorators(node, cls): + if not name in ('attr.s', 'attr.attrs', 'attr.attributes'): + continue + + attrs_args = get_attrs_args(decnode, cls.module) + if attrs_args: + cls.attrs_auto_attribs = uses_auto_attribs(attrs_args, decnode.lineno, cls.module) + cls.attrs_init = uses_init(attrs_args, decnode.lineno, cls.module) + cls.attrs_kw_only = uses_kw_only(attrs_args, decnode.lineno, cls.module) + break + + # since self.when = visitor.When.INNER, we can depart the classdef while still beeing + # inside it's context. + def depart_ClassDef(self, node:ast.ClassDef) -> None: + cls = self.visitor.builder.current + if not isinstance(cls, model.Class) or cls.name!=node.name: + return + assert isinstance(cls, AttrsClass) + + # by default attr.s() overrides any defined __init__ mehtod, whereas dataclasses. + # TODO: but if auto_detect=True, we need to check if __init__ already exists, otherwise it does not replace it. + # NOTE: But attr.define() use auto_detect=True by default! this is getting complicated... + if cls.attrs_init: + func = self.visitor.builder.pushFunction('__init__', node.lineno) + # init Function attributes that otherwise would be undefined :/ + func.decorators = None + func.is_async = False + + try: + func.signature = cls.attrs_constructor_signature_builder.get_signature() + func.annotations = cls.attrs_constructor_annotations + except ValueError as e: + func.report(f'could not deduce attrs class __init__ signature: {e}') + func.signature = inspect.Signature() + func.annotations = {} + finally: + self.visitor.builder.popFunction() def _handleAttrsAssignmentInClass(self, target:str, node: Union[ast.Assign, ast.AnnAssign]) -> None: cls = self.visitor.builder.current @@ -136,14 +217,53 @@ def _handleAttrsAssignmentInClass(self, target:str, node: Union[ast.Assign, ast. annotation = node.annotation if isinstance(node, ast.AnnAssign) else None - if is_attrib(node.value, cls) or ( - cls.auto_attribs and \ + is_attrs_attrib = is_attrib(node.value, cls) + is_attrs_auto_attrib = cls.attrs_auto_attribs and \ annotation is not None and \ - not astutils.is_using_typing_classvar(annotation, cls)): + not astutils.is_using_typing_classvar(annotation, cls) + + if is_attrs_attrib or is_attrs_auto_attrib: attr.kind = model.DocumentableKind.INSTANCE_VARIABLE - if annotation is None and node.value is not None: - attr.annotation = annotation_from_attrib(self.visitor, node.value, cls) + attrib_args = get_attrib_args(node.value, cls) + + if annotation is None and attrib_args is not None: + attr.annotation = annotation_from_attrib(attrib_args, cls) + + # Handle the auto-creation of the __init__ method. + if cls.attrs_init: + + if is_attrs_auto_attrib or (attrib_args and uses_init(attrib_args, cls.module, node.lineno)): + kind = inspect.Parameter.POSITIONAL_OR_KEYWORD + + if cls.attrs_kw_only or (attrib_args and uses_kw_only(attrib_args, cls.module, node.lineno)): + kind = inspect.Parameter.KEYWORD_ONLY + + attrs_default = ast.Constant(value=...) + + if is_attrs_auto_attrib: + attrs_default = node.value + + if is_factory_call(attrs_default, cls): + # Factory is not a default value stricly speaking, + # so we give up on trying to figure it out. + attrs_default = ast.Constant(value=...) + + elif attrib_args is not None: + attrs_default = default_from_attrib(attrib_args, cls) + + # attrs strips the leading underscores from the parameter names, + # since there is not such thing as a private parameter. + _init_param_name = attr.name.lstrip('_') + + cls.attrs_constructor_signature_builder.add_param( + _init_param_name, kind=kind, default=attrs_default, annotation=None + ) + if attrib_args is not None: + cls.attrs_constructor_annotations[_init_param_name] = \ + annotation_from_attrib(attrib_args, cls, for_init_method=True) or annotation + else: + cls.attrs_constructor_annotations[_init_param_name] = annotation def _handleAttrsAssignment(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: for dottedname in astutils.iterassign(node): @@ -163,11 +283,27 @@ class AttrsClass(extensions.ClassMixin, model.Class): def setup(self) -> None: super().setup() - self.auto_attribs: bool = False + + self.attrs_auto_attribs: bool = False """ L{True} if this class uses the C{auto_attribs} feature of the L{attrs } library to automatically convert annotated fields into attributes. """ + + self.attrs_kw_only: bool = False + """ + C{True} is this class uses C{kw_only} feature of L{attrs } library. + """ + + self.attrs_init: bool = False + """ + False if L{attrs } is not generating an __init__ method for this class. + """ + + # since the signatures doesnt include type annotations, we track them in a separate attribute. + self.attrs_constructor_signature_builder = astbuilder.SignatureBuilder(self) + self.attrs_constructor_signature_builder.add_param('self', inspect.Parameter.POSITIONAL_OR_KEYWORD,) + self.attrs_constructor_annotations: Dict[str, Optional[ast.expr]] = {'self':None} def setup_pydoctor_extension(r:extensions.ExtRegistrar) -> None: r.register_astbuilder_visitor(ModuleVisitor) diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 1d01fd8d2..3e611d1ae 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -1,7 +1,9 @@ from typing import Type -from pydoctor import model +from pydoctor import epydoc2stan, model from pydoctor.extensions import attrs +from pydoctor.stanutils import flatten_text +from pydoctor.templatewriter import pages from pydoctor.test import CapSys from pydoctor.test.test_astbuilder import fromText, AttrsSystem, type2str @@ -90,7 +92,7 @@ class C: ''', modname='test', systemcls=systemcls) C = mod.contents['C'] assert isinstance(C, attrs.AttrsClass) - assert C.auto_attribs == True + assert C.attrs_auto_attribs == True assert C.contents['a'].kind is model.DocumentableKind.INSTANCE_VARIABLE assert C.contents['b'].kind is model.DocumentableKind.INSTANCE_VARIABLE assert C.contents['c'].kind is model.DocumentableKind.CLASS_VARIABLE @@ -122,6 +124,28 @@ class C4: ... captured = capsys.readouterr().out assert captured == ( 'test:10: Invalid arguments for attr.s(): got an unexpected keyword argument "auto_attribzzz"\n' - 'test:13: Unable to figure out value for "auto_attribs" argument to attr.s(), maybe too complex\n' - 'test:16: Value for "auto_attribs" argument to attr.s() has type "int", expected "bool"\n' + 'test:13: Unable to figure out value for \'auto_attribs\' argument to attr.s(), maybe too complex\n' + 'test:16: Value for "auto_attribs" argument has type "int", expected "bool"\n' ) + +@attrs_systemcls_param +def test_attrs_init_method(systemcls: Type[model.System], capsys: CapSys) -> None: + src = '''\ + @attr.s + class C(object): + c = attr.ib(default=100) + x = attr.ib(default=1) + b = attr.ib(default=23) + + @attr.s + class D(C): + a = attr.ib(default=42) + x = attr.ib(default=2) + d = attr.ib(default=3.14) + ''' + mod = fromText(src, systemcls=systemcls) + C = mod.contents['C'] + constructor = C.contents['__init__'] + assert isinstance(constructor, model.Function) + assert epydoc2stan.format_constructor_short_text(constructor, forclass=C) == 'C(c, x, b)' + assert flatten_text(pages.format_signature(constructor)) == '(self, c=100, x=1, b=23)' From 379c63e6d40f086433c4747b9c862ae764f4f98a Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Mon, 31 Oct 2022 01:03:52 -0400 Subject: [PATCH 08/48] Fix issues --- pydoctor/astutils.py | 30 ++++++++- pydoctor/extensions/attrs.py | 125 +++++++++++------------------------ pydoctor/test/test_attrs.py | 10 ++- 3 files changed, 73 insertions(+), 92 deletions(-) diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 50b2aae30..2a48a3100 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -138,7 +138,7 @@ def bind_args(sig: Signature, call: ast.Call) -> BoundArguments: return sig.bind(*call.args, **kwargs) _T = TypeVar('_T') -def get_bound_literal(args:BoundArguments, name:str, typecheck:Type[_T]=object) -> Union[object, _T]: +def _get_literal_arg(args:BoundArguments, name:str, typecheck:Type[_T]=object) -> Union[object, _T]: """ Retreive the literal value of an argument from the L{BoundArguments}. Only works with purely literal values (no C{Name} or C{Attribute}). @@ -155,18 +155,42 @@ def get_bound_literal(args:BoundArguments, name:str, typecheck:Type[_T]=object) value = ast.literal_eval(auto_attribs_expr) except ValueError: message = ( - f'Unable to figure out value for {name!r} argument, maybe too complex', + f'Unable to figure out value for {name!r} argument, maybe too complex' ).replace("'", '"') raise ValueError(message) if not isinstance(value, typecheck): message = (f'Value for {name!r} argument ' - f'has type "{type(value).__name__}", expected {typecheck.__name__!r}', + f'has type "{type(value).__name__}", expected {typecheck.__name__!r}' ).replace("'", '"') raise ValueError(message) return value +def get_literal_arg(args:BoundArguments, name:str, default:_T, + typecheck:Type[_T], lineno:int, module: 'model.Module') -> _T: + """ + Get the value of the C{auto_attribs} argument passed to this L{attr.s()} call. + + @param args: The L{BoundArguments} instance. + @param name: The name of the argument + @param default: The default value of the argument, this value is returned + if the argument could not be found. + @param typecheck: The type of the literal value this argument is expected to have. + @param lineno: The lineumber of the callsite, usd for error reporting. + @param module: Module that contains the call, used for error reporting. + @return: The value of the argument if we can infer it, otherwise returns + the default value. + """ + try: + value = _get_literal_arg(args, name, typecheck) + except ValueError as e: + module.report(str(e), lineno_offset=lineno) + return default + if value is Parameter.empty: + # default value + return default + return value if sys.version_info[:2] >= (3, 8): # Since Python 3.8 "foo" is parsed as ast.Constant. diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 5226c7e90..116d03b61 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -4,6 +4,7 @@ """ import ast +import functools import inspect from typing import Dict, Optional, Union @@ -61,59 +62,9 @@ def get_attrib_args(expr: ast.expr, ctx: model.Documentable) -> Optional[inspect ) return None -def uses_init( - args:inspect.BoundArguments, - lineno: int, - ctx: model.Module, -) -> bool: - """ - Get the value of the C{init} argument passed to this L{attr.ib()}/L{attr.s()} call. - """ - try: - init = astutils.get_bound_literal(args, 'init', bool) - except ValueError as e: - ctx.report(str(e), lineno_offset=lineno) - return False - if init is inspect.Parameter.empty: - # default value for attr.ib(init) is True - return True - return init - -def uses_kw_only(args:inspect.BoundArguments, lineno:int, ctx: model.Module) -> bool: - """ - Get the value of the C{kw_only} argument passed to this L{attr.ib()}/L{attr.s()} call. - """ - try: - init = astutils.get_bound_literal(args, 'kw_only', bool) - except ValueError as e: - ctx.report(str(e), lineno_offset=lineno) - return False - if init is inspect.Parameter.empty: - # default value for attr.ib(kw_only) is True - return False - return init - -def uses_auto_attribs(args:inspect.BoundArguments, lineno:int, module: model.Module) -> bool: - """ - Get the value of the C{auto_attribs} argument passed to this L{attr.s()} call. - - @param call: AST of the call to L{attr.s()}. - This function will assume that L{attr.s()} is called without - verifying that. - @param module: Module that contains the call, used for error reporting. - @return: L{True} if L{True} is passed for C{auto_attribs}, - L{False} in all other cases: if C{auto_attribs} is not passed, - if an explicit L{False} is passed or if an error was reported. - """ - try: - value = astutils.get_bound_literal(args, 'auto_attribs', bool) - except ValueError as e: - module.report(str(e), lineno_offset=lineno) - return False - if value is inspect.Parameter.empty: - # default value is False for attr.s(auto_attribs) - return False - return value +uses_init = functools.partial(astutils.get_literal_arg, name='init', default=True, typecheck=bool) +uses_kw_only = functools.partial(astutils.get_literal_arg, name='kw_only', default=False, typecheck=bool) +uses_auto_attribs = functools.partial(astutils.get_literal_arg, name='auto_attribs', default=False, typecheck=bool) def annotation_from_attrib( args:inspect.BoundArguments, @@ -155,8 +106,6 @@ def default_from_attrib(args:inspect.BoundArguments, ctx: model.Documentable) -> return None class ModuleVisitor(extensions.ModuleVisitorExt): - - when = visitor.When.INNER def visit_ClassDef(self, node:ast.ClassDef) -> None: """ @@ -171,40 +120,16 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: if not name in ('attr.s', 'attr.attrs', 'attr.attributes'): continue + # True by default + cls.attrs_init = True + attrs_args = get_attrs_args(decnode, cls.module) if attrs_args: - cls.attrs_auto_attribs = uses_auto_attribs(attrs_args, decnode.lineno, cls.module) - cls.attrs_init = uses_init(attrs_args, decnode.lineno, cls.module) - cls.attrs_kw_only = uses_kw_only(attrs_args, decnode.lineno, cls.module) + cls.attrs_auto_attribs = uses_auto_attribs(args=attrs_args, lineno=decnode.lineno, module=cls.module) + cls.attrs_init = uses_init(args=attrs_args, lineno=decnode.lineno, module=cls.module) + cls.attrs_kw_only = uses_kw_only(args=attrs_args, lineno=decnode.lineno, module=cls.module) break - # since self.when = visitor.When.INNER, we can depart the classdef while still beeing - # inside it's context. - def depart_ClassDef(self, node:ast.ClassDef) -> None: - cls = self.visitor.builder.current - if not isinstance(cls, model.Class) or cls.name!=node.name: - return - assert isinstance(cls, AttrsClass) - - # by default attr.s() overrides any defined __init__ mehtod, whereas dataclasses. - # TODO: but if auto_detect=True, we need to check if __init__ already exists, otherwise it does not replace it. - # NOTE: But attr.define() use auto_detect=True by default! this is getting complicated... - if cls.attrs_init: - func = self.visitor.builder.pushFunction('__init__', node.lineno) - # init Function attributes that otherwise would be undefined :/ - func.decorators = None - func.is_async = False - - try: - func.signature = cls.attrs_constructor_signature_builder.get_signature() - func.annotations = cls.attrs_constructor_annotations - except ValueError as e: - func.report(f'could not deduce attrs class __init__ signature: {e}') - func.signature = inspect.Signature() - func.annotations = {} - finally: - self.visitor.builder.popFunction() - def _handleAttrsAssignmentInClass(self, target:str, node: Union[ast.Assign, ast.AnnAssign]) -> None: cls = self.visitor.builder.current assert isinstance(cls, AttrsClass) @@ -233,10 +158,10 @@ def _handleAttrsAssignmentInClass(self, target:str, node: Union[ast.Assign, ast. # Handle the auto-creation of the __init__ method. if cls.attrs_init: - if is_attrs_auto_attrib or (attrib_args and uses_init(attrib_args, cls.module, node.lineno)): + if is_attrs_auto_attrib or (attrib_args and uses_init(args=attrib_args, module=cls.module, lineno=node.lineno)): kind = inspect.Parameter.POSITIONAL_OR_KEYWORD - if cls.attrs_kw_only or (attrib_args and uses_kw_only(attrib_args, cls.module, node.lineno)): + if cls.attrs_kw_only or (attrib_args and uses_kw_only(args=attrib_args, module=cls.module, lineno=node.lineno)): kind = inspect.Parameter.KEYWORD_ONLY attrs_default = ast.Constant(value=...) @@ -256,6 +181,9 @@ def _handleAttrsAssignmentInClass(self, target:str, node: Union[ast.Assign, ast. # since there is not such thing as a private parameter. _init_param_name = attr.name.lstrip('_') + # TODO: Check if attrs defines a converter, if it does not, it's OK + # to deduce that the type of the argument is the same as type of the parameter. + # But actually, this might be a wrong assumption. cls.attrs_constructor_signature_builder.add_param( _init_param_name, kind=kind, default=attrs_default, annotation=None ) @@ -305,6 +233,29 @@ def setup(self) -> None: self.attrs_constructor_signature_builder.add_param('self', inspect.Parameter.POSITIONAL_OR_KEYWORD,) self.attrs_constructor_annotations: Dict[str, Optional[ast.expr]] = {'self':None} +def postProcess(system:model.System) -> None: + + for cls in list(system.objectsOfType(model.Class)): + # by default attr.s() overrides any defined __init__ mehtod, whereas dataclasses. + # TODO: but if auto_detect=True, we need to check if __init__ already exists, otherwise it does not replace it. + # NOTE: But attr.define() use auto_detect=True by default! this is getting complicated... + if cls.attrs_init: + func = system.Function(system, '__init__', cls) + system.addObject(func) + # init Function attributes that otherwise would be undefined :/ + func.decorators = None + func.is_async = False + + try: + # TODO: collect arguments from super classes attributes definitions. + func.signature = cls.attrs_constructor_signature_builder.get_signature() + func.annotations = cls.attrs_constructor_annotations + except ValueError as e: + func.report(f'could not deduce attrs class __init__ signature: {e}') + func.signature = inspect.Signature() + func.annotations = {} + def setup_pydoctor_extension(r:extensions.ExtRegistrar) -> None: r.register_astbuilder_visitor(ModuleVisitor) r.register_mixin(AttrsClass) + r.register_post_processor(postProcess) diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 3e611d1ae..38adc35a7 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -124,7 +124,7 @@ class C4: ... captured = capsys.readouterr().out assert captured == ( 'test:10: Invalid arguments for attr.s(): got an unexpected keyword argument "auto_attribzzz"\n' - 'test:13: Unable to figure out value for \'auto_attribs\' argument to attr.s(), maybe too complex\n' + 'test:13: Unable to figure out value for "auto_attribs" argument, maybe too complex\n' 'test:16: Value for "auto_attribs" argument has type "int", expected "bool"\n' ) @@ -137,14 +137,20 @@ class C(object): x = attr.ib(default=1) b = attr.ib(default=23) - @attr.s + @attr.s(init=False) class D(C): a = attr.ib(default=42) x = attr.ib(default=2) d = attr.ib(default=3.14) ''' mod = fromText(src, systemcls=systemcls) + assert capsys.readouterr().out == '' C = mod.contents['C'] + assert C.attrs_init == True + D = mod.contents['D'] + assert D.attrs_init == False + + assert isinstance(C, model.Class) constructor = C.contents['__init__'] assert isinstance(constructor, model.Function) assert epydoc2stan.format_constructor_short_text(constructor, forclass=C) == 'C(c, x, b)' From c5fc9b9c99ac4984b3cf48ee68b67dd5fa51950b Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Mon, 31 Oct 2022 01:33:54 -0400 Subject: [PATCH 09/48] try to fix mypy --- pydoctor/astutils.py | 2 +- pydoctor/extensions/attrs.py | 11 ++++++----- pydoctor/model.py | 1 + pydoctor/test/test_astbuilder.py | 3 ++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 2a48a3100..60a31f457 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -137,7 +137,7 @@ def bind_args(sig: Signature, call: ast.Call) -> BoundArguments: } return sig.bind(*call.args, **kwargs) -_T = TypeVar('_T') +_T = TypeVar('_T', bound=object) def _get_literal_arg(args:BoundArguments, name:str, typecheck:Type[_T]=object) -> Union[object, _T]: """ Retreive the literal value of an argument from the L{BoundArguments}. diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 116d03b61..ba24b64dc 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -46,12 +46,13 @@ def is_factory_call(expr: ast.expr, ctx: model.Documentable) -> bool: return isinstance(expr, ast.Call) and \ astutils.node2fullname(expr.func, ctx) in ('attrs.Factory', 'attr.Factory') -def get_attrib_args(expr: ast.expr, ctx: model.Documentable) -> Optional[inspect.BoundArguments]: +def get_attrib_args(expr: Optional[ast.expr], ctx: model.Documentable) -> Optional[inspect.BoundArguments]: """Get the arguments passed to an C{attr.ib} definition. @return: The arguments, or L{None} if C{expr} does not look like an C{attr.ib} definition or the arguments passed to it are invalid. """ if is_attrib(expr, ctx): + assert expr is not None try: return astutils.bind_args(attrib_signature, expr) except TypeError as ex: @@ -99,7 +100,7 @@ def default_from_attrib(args:inspect.BoundArguments, ctx: model.Documentable) -> if d is not None: if is_factory_call(d, ctx): return ast.Constant(value=...) - return d + return d # type:ignore elif f: # If a factory is defined, the default value is not obvious. return ast.Constant(value=...) else: @@ -164,12 +165,12 @@ def _handleAttrsAssignmentInClass(self, target:str, node: Union[ast.Assign, ast. if cls.attrs_kw_only or (attrib_args and uses_kw_only(args=attrib_args, module=cls.module, lineno=node.lineno)): kind = inspect.Parameter.KEYWORD_ONLY - attrs_default = ast.Constant(value=...) + attrs_default:Optional[ast.AST] = ast.Constant(value=...) if is_attrs_auto_attrib: attrs_default = node.value - if is_factory_call(attrs_default, cls): + if attrs_default and is_factory_call(attrs_default, cls): # Factory is not a default value stricly speaking, # so we give up on trying to figure it out. attrs_default = ast.Constant(value=...) @@ -235,7 +236,7 @@ def setup(self) -> None: def postProcess(system:model.System) -> None: - for cls in list(system.objectsOfType(model.Class)): + for cls in list(system.objectsOfType(AttrsClass)): # by default attr.s() overrides any defined __init__ mehtod, whereas dataclasses. # TODO: but if auto_detect=True, we need to check if __init__ already exists, otherwise it does not replace it. # NOTE: But attr.define() use auto_detect=True by default! this is getting complicated... diff --git a/pydoctor/model.py b/pydoctor/model.py index ac00480b2..bfc35ef9e 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -777,6 +777,7 @@ class Function(Inheritable): annotations: Mapping[str, Optional[ast.expr]] decorators: Optional[Sequence[ast.expr]] signature: Optional[Signature] + parent: CanContainImportsDocumentable def setup(self) -> None: super().setup() diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index deb57bee9..ef9c079e0 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -2116,7 +2116,8 @@ def test_prepend_package_real_path(systemcls: Type[model.System]) -> None: finally: systemcls.systemBuilder = _builderT_init -def getConstructorsText(cls: model.Class) -> str: +def getConstructorsText(cls:model.Documentable) -> str: + assert isinstance(cls, model.Class) return '\n'.join( epydoc2stan.format_constructor_short_text(c, cls) for c in cls.constructors) From a2389dcb692a60974dc4c936abc4e383ff6b0453 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 11 Jun 2023 06:51:28 -0400 Subject: [PATCH 10/48] Abstract out some of the core visiting code for dataclass like classes --- pydoctor/extensions/_dataclass_like.py | 65 +++++++++++++++++++ pydoctor/extensions/attrs.py | 88 ++++++++++---------------- 2 files changed, 100 insertions(+), 53 deletions(-) create mode 100644 pydoctor/extensions/_dataclass_like.py diff --git a/pydoctor/extensions/_dataclass_like.py b/pydoctor/extensions/_dataclass_like.py new file mode 100644 index 000000000..912422747 --- /dev/null +++ b/pydoctor/extensions/_dataclass_like.py @@ -0,0 +1,65 @@ +""" +Dataclass-like libraries are all alike: +- They transform class variables into instance variable un certain conditions. +- They autoamtically provides a constructor method without having to define __init__. + +More specifically +""" +import ast +from abc import abstractmethod, ABC +from typing import Optional, Union +from pydoctor import astutils +from pydoctor.model import Module, Attribute, Class, Documentable +from pydoctor.extensions import ModuleVisitorExt, ClassMixin + +class DataclasLikeClass(ClassMixin): + isDataclassLike:bool = False + +class DataclassLikeVisitor(ModuleVisitorExt, ABC): + + @abstractmethod + def isDataclassLike(self, cls:ast.ClassDef, mod:Module) -> bool: + """ + Whether L{transformClassVar} method should be called for each class variables + in this class. + """ + + @abstractmethod + def transformClassVar(self, cls:Class, attr:Attribute, + annotation:Optional[ast.expr], + value:Optional[ast.expr]) -> None: + """ + Transform this class variable into a instance variable. + This method is left abstract because it might not be as simple as setting:: + attr.kind = model.DocumentableKind.INSTANCE_VARIABLE + (but it also might be just that for the simpler cases) + """ + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + cls = self.visitor.builder._stack[-1].contents.get(node.name) + if not isinstance(cls, Class): + return + assert isinstance(cls, DataclasLikeClass) + cls.isDataclassLike = self.isDataclassLike(node, cls.module) + + def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: + current = self.visitor.builder.current + + for dottedname in astutils.iterassign(node): + if dottedname and len(dottedname)==1: + # We consider single name assignment only + if not isinstance(current, Class): + continue + assert isinstance(current, DataclasLikeClass) + if not current.isDataclassLike: + continue + target, = dottedname + attr: Optional[Documentable] = current.contents.get(target) + if not isinstance(attr, Attribute): + continue + if astutils.is_using_typing_classvar(attr.annotation, current): + continue + annotation = node.annotation if isinstance(node, ast.AnnAssign) else None + self.transformClassVar(current, attr, annotation, node.value) + + visit_AnnAssign = visit_Assign diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 50dccc7b6..2687f8319 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -6,9 +6,10 @@ import ast import inspect -from typing import Optional, Union +from typing import Optional from pydoctor import astbuilder, model, astutils, extensions +from pydoctor.extensions._dataclass_like import DataclasLikeClass, DataclassLikeVisitor import attr @@ -18,6 +19,12 @@ attrib_signature = inspect.signature(attr.ib) """Signature of the L{attr.ib} function for defining class attributes.""" +def is_attrs_deco(deco: ast.AST, module: model.Module) -> bool: + if isinstance(deco, ast.Call): + deco = deco.func + return astutils.node2fullname(deco, module) in ( + 'attr.s', 'attr.attrs', 'attr.attributes') + def uses_auto_attribs(call: ast.AST, module: model.Module) -> bool: """Does the given L{attr.s()} decoration contain C{auto_attribs=True}? @param call: AST of the call to L{attr.s()}. @@ -28,9 +35,9 @@ def uses_auto_attribs(call: ast.AST, module: model.Module) -> bool: L{False} in all other cases: if C{auto_attribs} is not passed, if an explicit L{False} is passed or if an error was reported. """ - if not isinstance(call, ast.Call): + if not is_attrs_deco(call, module): return False - if not astutils.node2fullname(call.func, module) in ('attr.s', 'attr.attrs', 'attr.attributes'): + if not isinstance(call, ast.Call): return False try: args = astutils.bind_args(attrs_decorator_signature, call) @@ -77,9 +84,8 @@ def attrib_args(expr: ast.expr, ctx: model.Documentable) -> Optional[inspect.Bou @return: The arguments, or L{None} if C{expr} does not look like an C{attr.ib} definition or the arguments passed to it are invalid. """ - if isinstance(expr, ast.Call) and astutils.node2fullname(expr.func, ctx) in ( - 'attr.ib', 'attr.attrib', 'attr.attr' - ): + if is_attrib(expr, ctx): + assert isinstance(expr, ast.Call) try: return astutils.bind_args(attrib_signature, expr) except TypeError as ex: @@ -91,7 +97,6 @@ def attrib_args(expr: ast.expr, ctx: model.Documentable) -> Optional[inspect.Bou return None def annotation_from_attrib( - self: astbuilder.ModuleVistor, expr: ast.expr, ctx: model.Documentable ) -> Optional[ast.expr]: @@ -111,63 +116,40 @@ def annotation_from_attrib( return astbuilder._infer_type(default) return None -class ModuleVisitor(extensions.ModuleVisitorExt): +class ModuleVisitor(DataclassLikeVisitor): def visit_ClassDef(self, node:ast.ClassDef) -> None: """ Called when a class definition is visited. """ - cls = self.visitor.builder.current - if not isinstance(cls, model.Class) or cls.name!=node.name: + super().visit_ClassDef(node) + + cls = self.visitor.builder._stack[-1].contents.get(node.name) + if not isinstance(cls, AttrsClass): return - assert isinstance(cls, AttrsClass) cls.auto_attribs = any(uses_auto_attribs(decnode, cls.module) for decnode in node.decorator_list) - - def _handleAttrsAssignmentInClass(self, target:str, node: Union[ast.Assign, ast.AnnAssign]) -> None: - cls = self.visitor.builder.current + + def transformClassVar(self, cls: model.Class, + attr: model.Attribute, + annotation:Optional[ast.expr], + value:Optional[ast.expr]) -> None: assert isinstance(cls, AttrsClass) - - attr: Optional[model.Documentable] = cls.contents.get(target) - if attr is None: - return - if not isinstance(attr, model.Attribute): - return - - annotation = node.annotation if isinstance(node, ast.AnnAssign) else None - - if is_attrib(node.value, cls) or ( - cls.auto_attribs and \ - annotation is not None and \ - not astutils.is_using_typing_classvar(annotation, cls)): - + if is_attrib(value, cls) or (cls.auto_attribs and annotation is not None): attr.kind = model.DocumentableKind.INSTANCE_VARIABLE - if annotation is None and node.value is not None: - attr.annotation = annotation_from_attrib(self.visitor, node.value, cls) - - def _handleAttrsAssignment(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: - for dottedname in astutils.iterassign(node): - if dottedname and len(dottedname)==1: - # Here, we consider single name assignment only - current = self.visitor.builder.current - if isinstance(current, model.Class): - self._handleAttrsAssignmentInClass( - dottedname[0], node - ) - - def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: - self._handleAttrsAssignment(node) - visit_AnnAssign = visit_Assign - -class AttrsClass(extensions.ClassMixin, model.Class): + if annotation is None and value is not None: + attr.annotation = annotation_from_attrib(value, cls) - def setup(self) -> None: - super().setup() - self.auto_attribs: bool = False - """ - L{True} if this class uses the C{auto_attribs} feature of the L{attrs} - library to automatically convert annotated fields into attributes. - """ + def isDataclassLike(self, cls:ast.ClassDef, mod:model.Module) -> bool: + return any(is_attrs_deco(dec, mod) for dec in cls.decorator_list) + +class AttrsClass(DataclasLikeClass, model.Class): + + auto_attribs: bool = False + """ + L{True} if this class uses the C{auto_attribs} feature of the L{attrs} + library to automatically convert annotated fields into attributes. + """ def setup_pydoctor_extension(r:extensions.ExtRegistrar) -> None: r.register_astbuilder_visitor(ModuleVisitor) From 36e36836786bef1231380c3fcf49b509e2f12b59 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 11 Jun 2023 07:30:03 -0400 Subject: [PATCH 11/48] Factor-out callable analysis inside attrs.py into functions in astutils.py --- pydoctor/astutils.py | 99 +++++++++++++++++++++++++++++++++++- pydoctor/extensions/attrs.py | 84 ++++++------------------------ pydoctor/test/test_attrs.py | 4 +- 3 files changed, 115 insertions(+), 72 deletions(-) diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 720349b73..b41a64d56 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -2,11 +2,12 @@ Various bits of reusable code related to L{ast.AST} node processing. """ +import enum import inspect import platform import sys from numbers import Number -from typing import Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union +from typing import Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Type, TypeVar, Union from inspect import BoundArguments, Signature import ast @@ -14,6 +15,9 @@ if TYPE_CHECKING: from pydoctor import model + from typing import Protocol, Literal +else: + Protocol = Literal = object # AST visitors @@ -402,4 +406,95 @@ def extract_docstring(node: ast.Str) -> Tuple[int, str]: - The docstring to be parsed, cleaned by L{inspect.cleandoc}. """ lineno = extract_docstring_linenum(node) - return lineno, inspect.cleandoc(node.s) \ No newline at end of file + return lineno, inspect.cleandoc(node.s) + +def safe_bind_args(sig:Signature, call: ast.AST, ctx: 'model.Module') -> Optional[inspect.BoundArguments]: + """ + Get the arguments passed to a call based on it's known signature. + + Report warning when L{bind_args} raises a L{TypeError}. + """ + if not isinstance(call, ast.Call): + return None + try: + return bind_args(sig, call) + except TypeError as ex: + message = str(ex).replace("'", '"') + call_dottedname = node2dottedname(call.func) + callable_name = f"{'.'.join(call_dottedname)}()" if call_dottedname else 'callable' + ctx.report( + f"Invalid arguments for {callable_name}: {message}", + lineno_offset=call.lineno + ) + return None + +class _V(enum.Enum): + NoValue = enum.auto() +_T = TypeVar('_T', bound=object) +def _get_literal_arg(args:BoundArguments, name:str, typecheck:Type[_T]) -> Union['Literal[_V.NoValue]', _T]: + """ + Retreive the literal value of an argument from the L{BoundArguments}. + Only works with purely literal values (no C{Name} or C{Attribute}). + If the value is not present in the arguments, returns L{_V.NoValue}. + @raises ValueError: If the passed value is not a literal or if it's not the right type. + """ + expr = args.arguments.get(name) + if expr is None: + return _V.NoValue + + try: + value = ast.literal_eval(expr) + except ValueError: + message = ( + f'Unable to figure out value for {name!r} argument, maybe too complex' + ).replace("'", '"') + raise ValueError(message) + + if not isinstance(value, typecheck): + message = (f'Value for {name!r} argument ' + f'has type "{type(value).__name__}", expected {typecheck.__name__!r}' + ).replace("'", '"') + raise ValueError(message) + + return value + +def get_literal_arg(args:BoundArguments, name:str, default:_T, + typecheck:Type[_T], lineno:int, module: 'model.Module') -> _T: + """ + Get the value of the C{auto_attribs} argument passed to this L{attr.s()} call. + + @param args: The L{BoundArguments} instance. + @param name: The name of the argument + @param default: The default value of the argument, this value is returned + if the argument could not be found. + @param typecheck: The type of the literal value this argument is expected to have. + @param lineno: The lineumber of the callsite, usd for error reporting. + @param module: Module that contains the call, used for error reporting. + @return: The value of the argument if we can infer it, otherwise returns + the default value. + """ + try: + value = _get_literal_arg(args, name, typecheck) + except ValueError as e: + module.report(str(e), lineno_offset=lineno) + return default + if value is _V.NoValue: + # default value + return default + else: + return value + +class _HasDecoratorList(Protocol): + decorator_list:List[ast.expr] + +def iter_decorators(node:_HasDecoratorList, ctx: 'model.Documentable') -> Iterator[Tuple[Optional[str], ast.AST]]: + """ + Utility function to iterate decorators. + """ + + for decnode in node.decorator_list: + namenode = decnode + if isinstance(namenode, ast.Call): + namenode = namenode.func + dottedname = node2fullname(namenode, ctx) + yield dottedname, decnode diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 2687f8319..1a0f9916a 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -25,77 +25,12 @@ def is_attrs_deco(deco: ast.AST, module: model.Module) -> bool: return astutils.node2fullname(deco, module) in ( 'attr.s', 'attr.attrs', 'attr.attributes') -def uses_auto_attribs(call: ast.AST, module: model.Module) -> bool: - """Does the given L{attr.s()} decoration contain C{auto_attribs=True}? - @param call: AST of the call to L{attr.s()}. - This function will assume that L{attr.s()} is called without - verifying that. - @param module: Module that contains the call, used for error reporting. - @return: L{True} if L{True} is passed for C{auto_attribs}, - L{False} in all other cases: if C{auto_attribs} is not passed, - if an explicit L{False} is passed or if an error was reported. - """ - if not is_attrs_deco(call, module): - return False - if not isinstance(call, ast.Call): - return False - try: - args = astutils.bind_args(attrs_decorator_signature, call) - except TypeError as ex: - message = str(ex).replace("'", '"') - module.report( - f"Invalid arguments for attr.s(): {message}", - lineno_offset=call.lineno - ) - return False - - auto_attribs_expr = args.arguments.get('auto_attribs') - if auto_attribs_expr is None: - return False - - try: - value = ast.literal_eval(auto_attribs_expr) - except ValueError: - module.report( - 'Unable to figure out value for "auto_attribs" argument ' - 'to attr.s(), maybe too complex', - lineno_offset=call.lineno - ) - return False - - if not isinstance(value, bool): - module.report( - f'Value for "auto_attribs" argument to attr.s() ' - f'has type "{type(value).__name__}", expected "bool"', - lineno_offset=call.lineno - ) - return False - - return value - def is_attrib(expr: Optional[ast.expr], ctx: model.Documentable) -> bool: """Does this expression return an C{attr.ib}?""" return isinstance(expr, ast.Call) and astutils.node2fullname(expr.func, ctx) in ( 'attr.ib', 'attr.attrib', 'attr.attr' ) -def attrib_args(expr: ast.expr, ctx: model.Documentable) -> Optional[inspect.BoundArguments]: - """Get the arguments passed to an C{attr.ib} definition. - @return: The arguments, or L{None} if C{expr} does not look like - an C{attr.ib} definition or the arguments passed to it are invalid. - """ - if is_attrib(expr, ctx): - assert isinstance(expr, ast.Call) - try: - return astutils.bind_args(attrib_signature, expr) - except TypeError as ex: - message = str(ex).replace("'", '"') - ctx.module.report( - f"Invalid arguments for attr.ib(): {message}", - lineno_offset=expr.lineno - ) - return None - def annotation_from_attrib( expr: ast.expr, ctx: model.Documentable @@ -106,7 +41,9 @@ def annotation_from_attrib( @return: A type annotation, or None if the expression is not an C{attr.ib} definition or contains no type information. """ - args = attrib_args(expr, ctx) + args = None + if is_attrib(expr, ctx): + args = astutils.safe_bind_args(attrib_signature, expr, ctx.module) if args is not None: typ = args.arguments.get('type') if typ is not None: @@ -125,10 +62,21 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: super().visit_ClassDef(node) cls = self.visitor.builder._stack[-1].contents.get(node.name) - if not isinstance(cls, AttrsClass): + if not isinstance(cls, AttrsClass) or not cls.isDataclassLike: return - cls.auto_attribs = any(uses_auto_attribs(decnode, cls.module) for decnode in node.decorator_list) + for name, decnode in astutils.iter_decorators(node, cls): + if not name in ('attr.s', 'attr.attrs', 'attr.attributes'): + continue + + attrs_args = astutils.safe_bind_args(attrs_decorator_signature, decnode, cls.module) + if attrs_args: + + cls.auto_attribs = astutils.get_literal_arg( + name='auto_attribs', default=False, typecheck=bool, + args=attrs_args, lineno=decnode.lineno, module=cls.module) + + break def transformClassVar(self, cls: model.Class, attr: model.Attribute, diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 1d01fd8d2..eff5d8949 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -122,6 +122,6 @@ class C4: ... captured = capsys.readouterr().out assert captured == ( 'test:10: Invalid arguments for attr.s(): got an unexpected keyword argument "auto_attribzzz"\n' - 'test:13: Unable to figure out value for "auto_attribs" argument to attr.s(), maybe too complex\n' - 'test:16: Value for "auto_attribs" argument to attr.s() has type "int", expected "bool"\n' + 'test:13: Unable to figure out value for "auto_attribs" argument, maybe too complex\n' + 'test:16: Value for "auto_attribs" argument has type "int", expected "bool"\n' ) From d974015b34efa4b9c6646fb1e73ba4a331381beb Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 11 Jun 2023 07:36:02 -0400 Subject: [PATCH 12/48] Fix docstrings --- pydoctor/astutils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index b41a64d56..556c5b44f 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -410,9 +410,9 @@ def extract_docstring(node: ast.Str) -> Tuple[int, str]: def safe_bind_args(sig:Signature, call: ast.AST, ctx: 'model.Module') -> Optional[inspect.BoundArguments]: """ - Get the arguments passed to a call based on it's known signature. + Binds the arguments of a function call to that function's signature. - Report warning when L{bind_args} raises a L{TypeError}. + When L{bind_args} raises a L{TypeError}, it reports a warning and returns C{None}. """ if not isinstance(call, ast.Call): return None @@ -433,8 +433,8 @@ class _V(enum.Enum): _T = TypeVar('_T', bound=object) def _get_literal_arg(args:BoundArguments, name:str, typecheck:Type[_T]) -> Union['Literal[_V.NoValue]', _T]: """ - Retreive the literal value of an argument from the L{BoundArguments}. - Only works with purely literal values (no C{Name} or C{Attribute}). + Helper function for L{get_literal_arg}. + If the value is not present in the arguments, returns L{_V.NoValue}. @raises ValueError: If the passed value is not a literal or if it's not the right type. """ @@ -461,12 +461,13 @@ def _get_literal_arg(args:BoundArguments, name:str, typecheck:Type[_T]) -> Union def get_literal_arg(args:BoundArguments, name:str, default:_T, typecheck:Type[_T], lineno:int, module: 'model.Module') -> _T: """ - Get the value of the C{auto_attribs} argument passed to this L{attr.s()} call. + Retreive the literal value of an argument from the L{BoundArguments}. + Only works with purely literal values (no C{Name} or C{Attribute}). @param args: The L{BoundArguments} instance. @param name: The name of the argument @param default: The default value of the argument, this value is returned - if the argument could not be found. + if the argument is not found. @param typecheck: The type of the literal value this argument is expected to have. @param lineno: The lineumber of the callsite, usd for error reporting. @param module: Module that contains the call, used for error reporting. From d1281cf6763aca3fc1c7fb7ad9ff89709b994ddf Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Sun, 11 Jun 2023 14:33:55 +0200 Subject: [PATCH 13/48] Update pydoctor/astutils.py --- pydoctor/astutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 556c5b44f..0af1c1d91 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -469,7 +469,7 @@ def get_literal_arg(args:BoundArguments, name:str, default:_T, @param default: The default value of the argument, this value is returned if the argument is not found. @param typecheck: The type of the literal value this argument is expected to have. - @param lineno: The lineumber of the callsite, usd for error reporting. + @param lineno: The lineumber of the callsite, used for error reporting. @param module: Module that contains the call, used for error reporting. @return: The value of the argument if we can infer it, otherwise returns the default value. From 41d8ddbf4986afd44a91fe0defdc841e39d99bf0 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 11 Jun 2023 13:40:40 -0400 Subject: [PATCH 14/48] Refactors --- pydoctor/astutils.py | 15 ------------- pydoctor/extensions/_dataclass_like.py | 5 ++--- pydoctor/extensions/attrs.py | 30 ++++++++++++-------------- 3 files changed, 16 insertions(+), 34 deletions(-) diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 556c5b44f..5de40750e 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -484,18 +484,3 @@ def get_literal_arg(args:BoundArguments, name:str, default:_T, return default else: return value - -class _HasDecoratorList(Protocol): - decorator_list:List[ast.expr] - -def iter_decorators(node:_HasDecoratorList, ctx: 'model.Documentable') -> Iterator[Tuple[Optional[str], ast.AST]]: - """ - Utility function to iterate decorators. - """ - - for decnode in node.decorator_list: - namenode = decnode - if isinstance(namenode, ast.Call): - namenode = namenode.func - dottedname = node2fullname(namenode, ctx) - yield dottedname, decnode diff --git a/pydoctor/extensions/_dataclass_like.py b/pydoctor/extensions/_dataclass_like.py index 912422747..90c308372 100644 --- a/pydoctor/extensions/_dataclass_like.py +++ b/pydoctor/extensions/_dataclass_like.py @@ -55,9 +55,8 @@ def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: continue target, = dottedname attr: Optional[Documentable] = current.contents.get(target) - if not isinstance(attr, Attribute): - continue - if astutils.is_using_typing_classvar(attr.annotation, current): + if not isinstance(attr, Attribute) or \ + astutils.is_using_typing_classvar(attr.annotation, current): continue annotation = node.annotation if isinstance(node, ast.AnnAssign) else None self.transformClassVar(current, attr, annotation, node.value) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 1a0f9916a..8fe9b06ef 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -41,9 +41,9 @@ def annotation_from_attrib( @return: A type annotation, or None if the expression is not an C{attr.ib} definition or contains no type information. """ - args = None - if is_attrib(expr, ctx): - args = astutils.safe_bind_args(attrib_signature, expr, ctx.module) + if not is_attrib(expr, ctx): + return None + args = astutils.safe_bind_args(attrib_signature, expr, ctx.module) if args is not None: typ = args.arguments.get('type') if typ is not None: @@ -64,19 +64,18 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: cls = self.visitor.builder._stack[-1].contents.get(node.name) if not isinstance(cls, AttrsClass) or not cls.isDataclassLike: return + mod = cls.module + try: + attrs_deco = next(decnode for decnode in node.decorator_list + if is_attrs_deco(decnode, mod)) + except StopIteration: + return - for name, decnode in astutils.iter_decorators(node, cls): - if not name in ('attr.s', 'attr.attrs', 'attr.attributes'): - continue - - attrs_args = astutils.safe_bind_args(attrs_decorator_signature, decnode, cls.module) - if attrs_args: - - cls.auto_attribs = astutils.get_literal_arg( - name='auto_attribs', default=False, typecheck=bool, - args=attrs_args, lineno=decnode.lineno, module=cls.module) - - break + attrs_args = astutils.safe_bind_args(attrs_decorator_signature, attrs_deco, mod) + if attrs_args: + cls.auto_attribs = astutils.get_literal_arg( + name='auto_attribs', default=False, typecheck=bool, + args=attrs_args, lineno=attrs_deco.lineno, module=mod) def transformClassVar(self, cls: model.Class, attr: model.Attribute, @@ -92,7 +91,6 @@ def isDataclassLike(self, cls:ast.ClassDef, mod:model.Module) -> bool: return any(is_attrs_deco(dec, mod) for dec in cls.decorator_list) class AttrsClass(DataclasLikeClass, model.Class): - auto_attribs: bool = False """ L{True} if this class uses the C{auto_attribs} feature of the L{attrs} From ba47bab80d3ba3c06283bc59b9134a95da538905 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 11 Jun 2023 13:43:27 -0400 Subject: [PATCH 15/48] Fix docstring --- pydoctor/extensions/_dataclass_like.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pydoctor/extensions/_dataclass_like.py b/pydoctor/extensions/_dataclass_like.py index 90c308372..a31bcf6c0 100644 --- a/pydoctor/extensions/_dataclass_like.py +++ b/pydoctor/extensions/_dataclass_like.py @@ -1,9 +1,7 @@ """ Dataclass-like libraries are all alike: - They transform class variables into instance variable un certain conditions. -- They autoamtically provides a constructor method without having to define __init__. - -More specifically +- They automatically provides a constructor method without having to define __init__. """ import ast from abc import abstractmethod, ABC From 9b3860b2bcfd7cb93ed850d053a756dd4750e79e Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 11 Jun 2023 17:28:21 -0400 Subject: [PATCH 16/48] remove commented code --- pydoctor/extensions/attrs.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index a89a81c6a..70e743bb0 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -52,17 +52,6 @@ def annotation_from_attrib( @return: A type annotation, or None if the expression is not an C{attr.ib} definition or contains no type information. """ - # if not is_attrib(expr, ctx): - # return None - # args = astutils.safe_bind_args(attrib_signature, expr, ctx.module) - # if args is not None: - # typ = args.arguments.get('type') - # if typ is not None: - # return astutils.unstring_annotation(typ, ctx) - # default = args.arguments.get('default') - # if default is not None: - # return astbuilder._infer_type(default) - # return None typ = args.arguments.get('type') if typ is not None: return astutils.unstring_annotation(typ, ctx) From a3f0cd3958a585432a96454181a2e67668f49f9a Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 11 Jun 2023 17:29:05 -0400 Subject: [PATCH 17/48] Remove unused imports --- pydoctor/extensions/attrs.py | 3 +-- pydoctor/test/test_attrs.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 70e743bb0..ebb2a0bc8 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -4,10 +4,9 @@ """ import ast -import functools import inspect -from typing import Dict, Optional, Union, cast +from typing import Dict, Optional, cast from pydoctor import astbuilder, model, astutils, extensions from pydoctor.extensions._dataclass_like import DataclasLikeClass, DataclassLikeVisitor diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 38adc35a7..beed389bf 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -146,8 +146,10 @@ class D(C): mod = fromText(src, systemcls=systemcls) assert capsys.readouterr().out == '' C = mod.contents['C'] + assert isinstance(C, attrs.AttrsClass) assert C.attrs_init == True D = mod.contents['D'] + assert isinstance(D, attrs.AttrsClass) assert D.attrs_init == False assert isinstance(C, model.Class) From 4822ea9e542bdde7d5c84f3acdf31b8b2ef2a7e1 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 11 Jun 2023 17:35:27 -0400 Subject: [PATCH 18/48] Fix annotation of signature_from_functiondef() --- pydoctor/astbuilder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index 8f34a5e8a..cd0d732bd 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -966,7 +966,7 @@ def __repr__(self) -> str: return ''.join(node2stan.node2html(self._colorized.to_node(), self._linker)) def signature_from_functiondef(node: Union[ast.AsyncFunctionDef, ast.FunctionDef], - ctx: model.Function) -> Signature: + ctx: model.Function) -> Tuple[Mapping[str, Optional[ast.expr]], Signature]: # Position-only arguments were introduced in Python 3.8. posonlyargs: Sequence[ast.arg] = getattr(node.args, 'posonlyargs', ()) From 3f8c7f649934ebfbb1f17bf53f7cdd7ace6965bf Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 13 Jun 2023 20:34:18 -0400 Subject: [PATCH 19/48] Improve support for attrs generated classes, still WIP... --- pydoctor/extensions/attrs.py | 90 +++++++++++++----- pydoctor/test/test_attrs.py | 173 ++++++++++++++++++++++++++++++++++- 2 files changed, 236 insertions(+), 27 deletions(-) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index ebb2a0bc8..ad000ce4a 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -6,7 +6,7 @@ import ast import inspect -from typing import Dict, Optional, cast +from typing import Dict, List, Optional, cast from pydoctor import astbuilder, model, astutils, extensions from pydoctor.extensions._dataclass_like import DataclasLikeClass, DataclassLikeVisitor @@ -97,12 +97,24 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: attrs_args = astutils.safe_bind_args(attrs_decorator_signature, attrs_deco, mod) if attrs_args: - cls.attrs_auto_attribs = astutils.get_literal_arg(name='auto_attribs', default=False, typecheck=bool, - args=attrs_args, lineno=attrs_deco.lineno, module=mod) - cls.attrs_init = astutils.get_literal_arg(name='init', default=True, typecheck=bool, - args=attrs_args, lineno=attrs_deco.lineno, module=mod) - cls.attrs_kw_only = astutils.get_literal_arg(name='kw_only', default=False, typecheck=bool, - args=attrs_args, lineno=attrs_deco.lineno, module=mod) + cls.attrs_auto_attribs = astutils.get_literal_arg(name='auto_attribs', + default=False, + typecheck=bool, + args=attrs_args, + lineno=attrs_deco.lineno, + module=mod) + cls.attrs_init = astutils.get_literal_arg(name='init', + default=True, + typecheck=bool, + args=attrs_args, + lineno=attrs_deco.lineno, + module=mod) + cls.attrs_kw_only = astutils.get_literal_arg(name='kw_only', + default=False, + typecheck=bool, + args=attrs_args, + lineno=attrs_deco.lineno, + module=mod) def transformClassVar(self, cls: model.Class, attr: model.Attribute, @@ -110,13 +122,14 @@ def transformClassVar(self, cls: model.Class, value:Optional[ast.expr]) -> None: assert isinstance(cls, AttrsClass) is_attrs_attrib = is_attrib(value, cls) - is_attrs_auto_attrib = cls.attrs_auto_attribs and annotation is not None + is_attrs_auto_attrib = cls.attrs_auto_attribs and not is_attrs_attrib and annotation is not None if is_attrs_attrib or is_attrs_auto_attrib: + attr.kind = model.DocumentableKind.INSTANCE_VARIABLE - if annotation is None and value is not None: + if value is not None: attrib_args = astutils.safe_bind_args(attrib_signature, value, cls.module) - if attrib_args: + if attrib_args and annotation is None: attr.annotation = annotation_from_attrib(attrib_args, cls) else: attrib_args = None @@ -125,13 +138,22 @@ def transformClassVar(self, cls: model.Class, if cls.attrs_init: if is_attrs_auto_attrib or (attrib_args and - astutils.get_literal_arg(name='init', default=True, typecheck=bool, - args=attrib_args, module=cls.module, lineno=attr.linenumber)): + astutils.get_literal_arg(name='init', + default=True, + typecheck=bool, + args=attrib_args, + module=cls.module, + lineno=attr.linenumber)): + kind:inspect._ParameterKind = inspect.Parameter.POSITIONAL_OR_KEYWORD if cls.attrs_kw_only or (attrib_args and - astutils.get_literal_arg(name='kw_only', default=False, typecheck=bool, - args=attrib_args, module=cls.module, lineno=attr.linenumber)): + astutils.get_literal_arg(name='kw_only', + default=False, + typecheck=bool, + args=attrib_args, + module=cls.module, + lineno=attr.linenumber)): kind = inspect.Parameter.KEYWORD_ONLY attrs_default:Optional[ast.expr] = ast.Constant(value=...) @@ -151,20 +173,22 @@ def transformClassVar(self, cls: model.Class, # since there is not such thing as a private parameter. init_param_name = attr.name.lstrip('_') + if attrib_args is not None: + constructor_annotation = cls.attrs_constructor_annotations[init_param_name] = \ + annotation_from_attrib(attrib_args, cls, for_init_method=True) or annotation + else: + constructor_annotation = cls.attrs_constructor_annotations[init_param_name] = annotation + # TODO: Check if attrs defines a converter, if it does not, it's OK # to deduce that the type of the argument is the same as type of the parameter. # But actually, this might be a wrong assumption. cls.attrs_constructor_parameters.append( inspect.Parameter( init_param_name, kind=kind, - default=astbuilder._ValueFormatter(attrs_default, cls) if attrs_default else inspect.Parameter.empty, - annotation=inspect.Parameter.empty)) - - if attrib_args is not None: - cls.attrs_constructor_annotations[init_param_name] = \ - annotation_from_attrib(attrib_args, cls, for_init_method=True) or annotation - else: - cls.attrs_constructor_annotations[init_param_name] = annotation + default=astbuilder._ValueFormatter(attrs_default, cls) + if attrs_default else inspect.Parameter.empty, + annotation=astbuilder._AnnotationValueFormatter(constructor_annotation, cls) + if constructor_annotation else inspect.Parameter.empty)) def isDataclassLike(self, cls:ast.ClassDef, mod:model.Module) -> bool: return any(is_attrs_deco(dec, mod) for dec in cls.decorator_list) @@ -190,7 +214,7 @@ def setup(self) -> None: """ # since the signatures doesnt include type annotations, we track them in a separate attribute. - self.attrs_constructor_parameters = [] + self.attrs_constructor_parameters:List[inspect.Parameter] = [] self.attrs_constructor_parameters.append(inspect.Parameter('self', inspect.Parameter.POSITIONAL_OR_KEYWORD,)) self.attrs_constructor_annotations: Dict[str, Optional[ast.expr]] = {'self': None} @@ -206,15 +230,31 @@ def postProcess(system:model.System) -> None: # init Function attributes that otherwise would be undefined :/ func.decorators = None func.is_async = False + func.parentMod = cls.parentMod + func.setLineNumber(cls.linenumber) - func.annotations = cls.attrs_constructor_annotations + parameters = cls.attrs_constructor_parameters + annotations = cls.attrs_constructor_annotations + + # Re-ordering kw_only arguments at the end of the list + for param in tuple(parameters): + if param.kind is inspect.Parameter.KEYWORD_ONLY: + parameters.remove(param) + parameters.append(param) + ann = annotations[param.name] + del annotations[param.name] + annotations[param.name] = ann + + func.annotations = annotations try: # TODO: collect arguments from super classes attributes definitions. - func.signature = inspect.Signature(cls.attrs_constructor_parameters) + func.signature = inspect.Signature(parameters) except Exception as e: func.report(f'could not deduce attrs class __init__ signature: {e}') func.signature = inspect.Signature() func.annotations = {} + else: + cls.constructors.append(func) def setup_pydoctor_extension(r:extensions.ExtRegistrar) -> None: r.register_astbuilder_visitor(ModuleVisitor) diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index beed389bf..89b012634 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -129,7 +129,7 @@ class C4: ... ) @attrs_systemcls_param -def test_attrs_init_method(systemcls: Type[model.System], capsys: CapSys) -> None: +def test_attrs_constructor_method_infer_arg_types(systemcls: Type[model.System], capsys: CapSys) -> None: src = '''\ @attr.s class C(object): @@ -156,4 +156,173 @@ class D(C): constructor = C.contents['__init__'] assert isinstance(constructor, model.Function) assert epydoc2stan.format_constructor_short_text(constructor, forclass=C) == 'C(c, x, b)' - assert flatten_text(pages.format_signature(constructor)) == '(self, c=100, x=1, b=23)' + assert flatten_text(pages.format_signature(constructor)) == '(self, c: int = 100, x: int = 1, b: int = 23)' + +# Test case for auto_attribs with defaults +@attrs_systemcls_param +def test_attrs_constructor_auto_attribs(systemcls: Type[model.System]) -> None: + src = '''\ + import attr + @attr.s(auto_attribs=True) + class C: + a: int + b: str = "default" + ''' + mod = fromText(src, systemcls=systemcls) + C = mod.contents['C'] + assert isinstance(C, model.Class) + constructor = C.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str = \'default\')' + +# Test case for kw_only +@attrs_systemcls_param +def test_attrs_constructor_kw_only(systemcls: Type[model.System]) -> None: + src = '''\ + import attr + @attr.s(kw_only=True) + class C: + a = attr.ib() + b: str = attr.ib() + ''' + mod = fromText(src, systemcls=systemcls) + C = mod.contents['C'] + assert isinstance(C, attrs.AttrsClass) + assert C.attrs_kw_only==True + assert C.attrs_init==True + constructor = C.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self, *, a, b: str)' + +# Test case for default factory +@attrs_systemcls_param +def test_attrs_constructor_factory_optional(systemcls: Type[model.System]) -> None: + src = '''\ + import attr + @attr.s + class C: + a: int = attr.ib(factory=list) + b: str = attr.ib(factory=str) + ''' + mod = fromText(src, systemcls=systemcls) + C = mod.contents['C'] + assert isinstance(C, model.Class) + constructor = C.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self, a: int = list(), b: str = str())' + +# Test case for init=False: +@attrs_systemcls_param +def test_attrs_no_constructor(systemcls: Type[model.System]) -> None: + src = '''\ + import attr + @attr.s(init=False) + class C: + a: int = attr.ib() + b: str = attr.ib() + ''' + mod = fromText(src, systemcls=systemcls) + C = mod.contents['C'] + assert C.contents.get('__init__') is None + +# Test case for single inheritance: +@attrs_systemcls_param +def test_attrs_constructor_single_inheritance(systemcls: Type[model.System]) -> None: + src = '''\ + import attr + @attr.s(auto_attribs=True) + class Base: + a: int + + @attr.s(auto_attribs=True) + class Derived(Base): + b: str + ''' + mod = fromText(src, systemcls=systemcls) + Derived = mod.contents['Derived'] + assert isinstance(Derived, model.Class) + constructor = Derived.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str)' + +# Test case for multiple inheritance: +@attrs_systemcls_param +def test_attrs_constructor_multiple_inheritance(systemcls: Type[model.System]) -> None: + src = '''\ + import attr + @attr.s(auto_attribs=True) + class Base1: + a: int + + @attr.s(auto_attribs=True) + class Base2: + b: str + + @attr.s(auto_attribs=True) + class Derived(Base1, Base2): + c: float + ''' + mod = fromText(src, systemcls=systemcls) + Derived = mod.contents['Derived'] + assert isinstance(Derived, model.Class) + constructor = Derived.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str, c: float)' + +# Test case for inheritance with overridden attributes: +@attrs_systemcls_param +def test_attrs_constructor_single_inheritance_overridden_attribute(systemcls: Type[model.System]) -> None: + src = '''\ + import attr + @attr.s(auto_attribs=True) + class Base: + a: int + b: str = "default" + + @attr.s(auto_attribs=True) + class Derived(Base): + b: str = "overridden" + c: float = 3.14 + ''' + mod = fromText(src, systemcls=systemcls) + Derived = mod.contents['Derived'] + assert isinstance(Derived, model.Class) + constructor = Derived.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str = "overridden", c: float = 3.14)' + +# Test case with attr.ib(init=False): +@attrs_systemcls_param +def test_attrs_constructor_attribute_init_False(systemcls: Type[model.System]) -> None: + src = '''\ + import attr + @attr.s(auto_attribs=True) + class MyClass: + a: int + b: str = attr.ib(init=False) + ''' + mod = fromText(src, systemcls=systemcls) + MyClass = mod.contents['MyClass'] + assert isinstance(MyClass, model.Class) + constructor = MyClass.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self, a: int)' + +# Test case with attr.ib(kw_only=True): +@attrs_systemcls_param +def test_attrs_constructor_attribute_kw_only_reorder(systemcls: Type[model.System], capsys:CapSys) -> None: + src = '''\ + import attr + @attr.s(auto_attribs=True) + class MyClass: + a: int + b: str = attr.ib(kw_only=True) + c: float + ''' + mod = fromText(src, systemcls=systemcls) + assert not capsys.readouterr().out + MyClass = mod.contents['MyClass'] + assert isinstance(MyClass, model.Class) + constructor = MyClass.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, c: float, *, b: str)' From 31ef26806cf249ef9cde0d2b6b0db0a47fa8a63a Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 16 Jun 2023 11:01:17 -0400 Subject: [PATCH 20/48] Better understand the factory parameter. Include the factory call as default value when possible. --- pydoctor/extensions/attrs.py | 112 ++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 54 deletions(-) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index ad000ce4a..c4b111086 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -31,12 +31,20 @@ def is_attrib(expr: Optional[ast.expr], ctx: model.Documentable) -> bool: 'attr.ib', 'attr.attrib', 'attr.attr' ) -def is_factory_call(expr: ast.expr, ctx: model.Documentable) -> bool: +def get_factory(expr: Optional[ast.expr], ctx: model.Documentable) -> Optional[ast.expr]: """ - Does this AST represent a call to L{attr.Factory}? + If this AST represent a call to L{attr.Factory}, returns the expression inside the factory call """ - return isinstance(expr, ast.Call) and \ - astutils.node2fullname(expr.func, ctx) in ('attrs.Factory', 'attr.Factory') + if isinstance(expr, ast.Call) and \ + astutils.node2fullname(expr.func, ctx) in ('attrs.Factory', 'attr.Factory'): + try: + factory, = expr.args + except Exception: + return None + else: + return factory + else: + return None def annotation_from_attrib( args:inspect.BoundArguments, @@ -68,12 +76,21 @@ def annotation_from_attrib( def default_from_attrib(args:inspect.BoundArguments, ctx: model.Documentable) -> Optional[ast.expr]: d = args.arguments.get('default') f = args.arguments.get('factory') - if d is not None: - if is_factory_call(d, ctx): - return ast.Constant(value=...) - return cast(ast.expr, d) - elif f: # If a factory is defined, the default value is not obvious. - return ast.Constant(value=...) + if isinstance(d, ast.expr): + factory = get_factory(d, ctx) + if factory: + if astutils.node2dottedname(factory): + return ast.Call(func=factory, args=[], keywords=[], lineno=d.lineno, col_offset=d.col_offset) + else: + return ast.Constant(value=..., lineno=d.lineno, col_offset=d.col_offset) + return d + elif isinstance(f, ast.expr): + if astutils.node2dottedname(f): + # If a simple factory is defined, the default value is a call to this function + return ast.Call(func=f, args=[], keywords=[], lineno=f.lineno, col_offset=f.col_offset) + else: + # Else we can't figure it out + return ast.Constant(value=..., lineno=f.lineno, col_offset=f.col_offset) else: return None @@ -97,24 +114,15 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: attrs_args = astutils.safe_bind_args(attrs_decorator_signature, attrs_deco, mod) if attrs_args: - cls.attrs_auto_attribs = astutils.get_literal_arg(name='auto_attribs', - default=False, - typecheck=bool, - args=attrs_args, - lineno=attrs_deco.lineno, - module=mod) - cls.attrs_init = astutils.get_literal_arg(name='init', - default=True, - typecheck=bool, - args=attrs_args, - lineno=attrs_deco.lineno, - module=mod) - cls.attrs_kw_only = astutils.get_literal_arg(name='kw_only', - default=False, - typecheck=bool, - args=attrs_args, - lineno=attrs_deco.lineno, - module=mod) + attrs_args_value = {name: astutils.get_literal_arg(attrs_args, name, default, + typecheck, attrs_deco.lineno, mod + ) for name, default, typecheck in + (('auto_attribs', False, bool), + ('init', True, bool), + ('kw_only', False, bool),)} + cls.attrs_auto_attribs = attrs_args_value['auto_attribs'] + cls.attrs_init = attrs_args_value['init'] + cls.attrs_kw_only = attrs_args_value['kw_only'] def transformClassVar(self, cls: model.Class, attr: model.Attribute, @@ -124,47 +132,43 @@ def transformClassVar(self, cls: model.Class, is_attrs_attrib = is_attrib(value, cls) is_attrs_auto_attrib = cls.attrs_auto_attribs and not is_attrs_attrib and annotation is not None + attrib_args = None + attrib_args_value = {} if is_attrs_attrib or is_attrs_auto_attrib: attr.kind = model.DocumentableKind.INSTANCE_VARIABLE if value is not None: attrib_args = astutils.safe_bind_args(attrib_signature, value, cls.module) - if attrib_args and annotation is None: - attr.annotation = annotation_from_attrib(attrib_args, cls) - else: - attrib_args = None + if attrib_args: + if annotation is None: + attr.annotation = annotation_from_attrib(attrib_args, cls) + + attrib_args_value = {name: astutils.get_literal_arg(attrib_args, name, default, + typecheck, attr.linenumber, cls.module + ) for name, default, typecheck in + (('init', True, bool), + ('kw_only', False, bool),)} # Handle the auto-creation of the __init__ method. if cls.attrs_init: - - if is_attrs_auto_attrib or (attrib_args and - astutils.get_literal_arg(name='init', - default=True, - typecheck=bool, - args=attrib_args, - module=cls.module, - lineno=attr.linenumber)): - + if is_attrs_auto_attrib or (attrib_args and attrib_args_value['init']): kind:inspect._ParameterKind = inspect.Parameter.POSITIONAL_OR_KEYWORD - - if cls.attrs_kw_only or (attrib_args and - astutils.get_literal_arg(name='kw_only', - default=False, - typecheck=bool, - args=attrib_args, - module=cls.module, - lineno=attr.linenumber)): + + if cls.attrs_kw_only or (attrib_args and attrib_args_value['kw_only']): kind = inspect.Parameter.KEYWORD_ONLY attrs_default:Optional[ast.expr] = ast.Constant(value=...) if is_attrs_auto_attrib: attrs_default = value - - if attrs_default and is_factory_call(attrs_default, cls): - # Factory is not a default value stricly speaking, - # so we give up on trying to figure it out. - attrs_default = ast.Constant(value=...) + factory = get_factory(attrs_default, cls) + if factory: + if astutils.node2dottedname(factory): + attrs_default = ast.Call(func=factory, args=[], keywords=[], lineno=factory.lineno, col_offset=factory.col_offset) + else: + # Factory is not a default value stricly speaking, + # so we give up on trying to figure it out. + attrs_default = ast.Constant(value=..., lineno=factory.lineno, col_offset=factory.col_offset) elif attrib_args is not None: attrs_default = default_from_attrib(attrib_args, cls) From c06de7e7c27008c456b0ea7cd2c61660518ca341 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 6 Jul 2023 15:48:40 -0400 Subject: [PATCH 21/48] Fix detected regression in overload handling --- pydoctor/astbuilder.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index cd0d732bd..433bc44b9 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -846,8 +846,6 @@ def _handleFunctionDef(self, func.kind = model.DocumentableKind.CLASS_METHOD annotations, signature = signature_from_functiondef(node, func) - - func.signature = signature func.annotations = annotations # Only set main function signature if it is a non-overload @@ -1019,7 +1017,7 @@ class _AnnotationValueFormatter(_ValueFormatter): """ Special L{_ValueFormatter} for function annotations. """ - def __init__(self, value: ast.expr, ctx: model.Function): + def __init__(self, value: ast.expr, ctx: model.Documentable): super().__init__(value, ctx) self._linker = linker._AnnotationLinker(ctx) From 2155fec49d7840327b253dad5c262a726db3fcf6 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 7 Jul 2023 10:01:52 -0400 Subject: [PATCH 22/48] Improve annotation_from_attrib() --- pydoctor/astutils.py | 11 ++++ pydoctor/extensions/attrs.py | 98 +++++++++++++++++++++++++++++------- pydoctor/test/test_attrs.py | 26 ++++++++-- 3 files changed, 113 insertions(+), 22 deletions(-) diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index be92862a1..504544dea 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -120,6 +120,17 @@ def node2dottedname(node: Optional[ast.AST]) -> Optional[List[str]]: parts.reverse() return parts +def dottedname2node(parts:List[str]) -> Union[ast.Name, ast.Attribute]: + """ + Reverse operation of L{node2dottedname}. + """ + assert parts, "must not be empty" + + if len(parts)==1: + return ast.Name(parts[0], ast.Load()) + else: + return ast.Attribute(dottedname2node(parts[:-1]), parts[-1], ast.Load()) + def node2fullname(expr: Optional[ast.AST], ctx: 'model.Documentable') -> Optional[str]: dottedname = node2dottedname(expr) if dottedname is None: diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index c4b111086..738381526 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -19,6 +19,10 @@ attrib_signature = inspect.signature(attr.ib) """Signature of the L{attr.ib} function for defining class attributes.""" +builtin_types = frozenset(('frozenset', 'int', 'bytes', + 'complex', 'list', 'tuple', + 'set', 'dict', 'range')) + def is_attrs_deco(deco: ast.AST, module: model.Module) -> bool: if isinstance(deco, ast.Call): deco = deco.func @@ -46,31 +50,90 @@ def get_factory(expr: Optional[ast.expr], ctx: model.Documentable) -> Optional[a else: return None +def _callable_return_type(dname:List[str], ctx:model.Documentable) -> Optional[ast.expr]: + """ + Given a callable dotted name in a certain context, + get it's return type as ast expression. + + Note that the expression might not be fully + resolvable in the new context since it can come from other modules. + """ + r = ctx.resolveName('.'.join(dname)) + if isinstance(r, model.Class): + return astutils.dottedname2node(dname) + elif isinstance(r, model.Function): + rtype = r.annotations.get('return') + if rtype: + return rtype + elif r is None and len(dname)==1 and dname[0] in builtin_types: + return astutils.dottedname2node(dname) + # TODO: we might be able to use the shpinx inventory yo check if the + # provided callable is a class, in which case the class could be linked. + return None + +def _annotation_from_factory( + factory:ast.expr, + ctx: model.Documentable, + ) -> Optional[ast.expr]: + dname = astutils.node2dottedname(factory) + if dname: + return _callable_return_type(dname, ctx) + else: + return None + +def _annotation_from_converter( + converter:ast.expr, + ctx: model.Documentable, + ) -> Optional[ast.expr]: + dname = astutils.node2dottedname(converter) + if dname: + r = ctx.resolveName('.'.join(dname)) + if isinstance(r, model.Class): + args = dict(r.constructor_params) + elif isinstance(r, model.Function): + args = dict(r.annotations) + else: + return None + args.pop('return', None) + if len(args)==1: + return args.popitem()[1] + return None + def annotation_from_attrib( args:inspect.BoundArguments, ctx: model.Documentable, - for_init_method:bool=False + for_constructor:bool=False ) -> Optional[ast.expr]: """Get the type of an C{attr.ib} definition. @param args: The L{inspect.BoundArguments} of the C{attr.ib()} call. @param ctx: The context in which this expression is evaluated. - @param for_init_method: Whether we're trying to figure out the __init__ parameter annotations + @param for_constructor: Whether we're trying to figure out the __init__ parameter annotations instead of the attribute annotations. @return: A type annotation, or None if the expression is not an C{attr.ib} definition or contains no type information. """ + if for_constructor: + # If a converter is defined... + converter = args.arguments.get('converter') + if converter is not None: + return _annotation_from_converter(converter, ctx) + typ = args.arguments.get('type') if typ is not None: return astutils.unstring_annotation(typ, ctx) + + factory = args.arguments.get('factory') + if factory is not None: + return _annotation_from_factory(factory, ctx) + default = args.arguments.get('default') if default is not None: - return astbuilder._infer_type(default) - # TODO: support factory parameter. - if for_init_method: - # If a converter is defined, then we can't be sure of what exact type of parameter is accepted - converter = args.arguments.get('converter') - if converter is not None: - return ast.Constant(value=...) + factory = get_factory(default, ctx) + if factory is not None: + return _annotation_from_factory(factory, ctx) + else: + return astbuilder._infer_type(default) + return None def default_from_attrib(args:inspect.BoundArguments, ctx: model.Documentable) -> Optional[ast.expr]: @@ -164,28 +227,27 @@ def transformClassVar(self, cls: model.Class, factory = get_factory(attrs_default, cls) if factory: if astutils.node2dottedname(factory): - attrs_default = ast.Call(func=factory, args=[], keywords=[], lineno=factory.lineno, col_offset=factory.col_offset) + attrs_default = ast.Call(func=factory, args=[], keywords=[], + lineno=factory.lineno, col_offset=factory.col_offset) else: # Factory is not a default value stricly speaking, # so we give up on trying to figure it out. - attrs_default = ast.Constant(value=..., lineno=factory.lineno, col_offset=factory.col_offset) + attrs_default = ast.Constant(value=..., lineno=factory.lineno, + col_offset=factory.col_offset) - elif attrib_args is not None: + elif attrib_args: attrs_default = default_from_attrib(attrib_args, cls) # attrs strips the leading underscores from the parameter names, # since there is not such thing as a private parameter. init_param_name = attr.name.lstrip('_') - if attrib_args is not None: + if attrib_args: constructor_annotation = cls.attrs_constructor_annotations[init_param_name] = \ - annotation_from_attrib(attrib_args, cls, for_init_method=True) or annotation + annotation_from_attrib(attrib_args, cls, for_constructor=True) or annotation else: constructor_annotation = cls.attrs_constructor_annotations[init_param_name] = annotation - # TODO: Check if attrs defines a converter, if it does not, it's OK - # to deduce that the type of the argument is the same as type of the parameter. - # But actually, this might be a wrong assumption. cls.attrs_constructor_parameters.append( inspect.Parameter( init_param_name, kind=kind, @@ -228,7 +290,7 @@ def postProcess(system:model.System) -> None: # by default attr.s() overrides any defined __init__ mehtod, whereas dataclasses. # TODO: but if auto_detect=True, we need to check if __init__ already exists, otherwise it does not replace it. # NOTE: But attr.define() use auto_detect=True by default! this is getting complicated... - if cls.attrs_init: + if cls.isDataclassLike and cls.attrs_init: func = system.Function(system, '__init__', cls) system.addObject(func) # init Function attributes that otherwise would be undefined :/ diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 89b012634..4b5e9b937 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -196,20 +196,38 @@ class C: # Test case for default factory @attrs_systemcls_param -def test_attrs_constructor_factory_optional(systemcls: Type[model.System]) -> None: +def test_attrs_constructor_factory(systemcls: Type[model.System]) -> None: src = '''\ import attr - @attr.s + @attr.s(auto_attribs=True) class C: a: int = attr.ib(factory=list) - b: str = attr.ib(factory=str) + b: str = attr.Factory(str) + c: list = attr.ib(default=attr.Factory(list)) ''' mod = fromText(src, systemcls=systemcls) C = mod.contents['C'] assert isinstance(C, model.Class) constructor = C.contents['__init__'] assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self, a: int = list(), b: str = str())' + assert flatten_text(pages.format_signature(constructor)) == '(self, a: list = list(), b: str = str(), c: list = list())' + +@attrs_systemcls_param +def test_attrs_constructor_factory_no_annotations(systemcls: Type[model.System]) -> None: + src = '''\ + import attr + @attr.s + class C: + a = attr.ib(factory=list) + b = attr.ib(default=attr.Factory(list)) + ''' + mod = fromText(src, systemcls=systemcls) + C = mod.contents['C'] + assert isinstance(C, model.Class) + constructor = C.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self, a: list = list(), b: list = list())' + # Test case for init=False: @attrs_systemcls_param From 4b9142e4ea84c189bc6baaa98a3b2cdb0b03f36b Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 7 Jul 2023 10:53:08 -0400 Subject: [PATCH 23/48] Better handle multiple dataclass like extensions --- pydoctor/extensions/_dataclass_like.py | 29 +++++++++++++++++++------- pydoctor/extensions/attrs.py | 10 ++++++--- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/pydoctor/extensions/_dataclass_like.py b/pydoctor/extensions/_dataclass_like.py index a31bcf6c0..cbd9bf29c 100644 --- a/pydoctor/extensions/_dataclass_like.py +++ b/pydoctor/extensions/_dataclass_like.py @@ -11,15 +11,26 @@ from pydoctor.extensions import ModuleVisitorExt, ClassMixin class DataclasLikeClass(ClassMixin): - isDataclassLike:bool = False + dataclassLike:Optional[object] = None class DataclassLikeVisitor(ModuleVisitorExt, ABC): + + DATACLASS_LIKE_KIND:object = NotImplemented + + def __init__(self) -> None: + super().__init__() + assert self.DATACLASS_LIKE_KIND is not NotImplemented, "constant DATACLASS_LIKE_KIND should have a value" @abstractmethod - def isDataclassLike(self, cls:ast.ClassDef, mod:Module) -> bool: + def isDataclassLike(self, cls:ast.ClassDef, mod:Module) -> Optional[object]: """ - Whether L{transformClassVar} method should be called for each class variables + If this classdef adopts dataclass-like behaviour, returns an non-zero int, otherwise returns None. + Returned value is directly stored in the C{dataclassLike} attribute of the visited class. + Used to determine whether L{transformClassVar} method should be called for each class variables in this class. + + The int value should be a constant representing the kind of dataclass-like this class implements. + Class decorated with @dataclass and @attr.s will have different non-zero C{dataclassLike} attribute. """ @abstractmethod @@ -28,9 +39,8 @@ def transformClassVar(self, cls:Class, attr:Attribute, value:Optional[ast.expr]) -> None: """ Transform this class variable into a instance variable. - This method is left abstract because it might not be as simple as setting:: + This method is left abstract because it's not as simple as setting:: attr.kind = model.DocumentableKind.INSTANCE_VARIABLE - (but it also might be just that for the simpler cases) """ def visit_ClassDef(self, node: ast.ClassDef) -> None: @@ -38,7 +48,12 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None: if not isinstance(cls, Class): return assert isinstance(cls, DataclasLikeClass) - cls.isDataclassLike = self.isDataclassLike(node, cls.module) + dataclassLikeKind = self.isDataclassLike(node, cls.module) + if dataclassLikeKind: + if not cls.dataclassLike: + cls.dataclassLike = dataclassLikeKind + else: + cls.report(f'class is both {cls.dataclassLike} and {dataclassLikeKind}') def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: current = self.visitor.builder.current @@ -49,7 +64,7 @@ def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: if not isinstance(current, Class): continue assert isinstance(current, DataclasLikeClass) - if not current.isDataclassLike: + if not current.dataclassLike == self.DATACLASS_LIKE_KIND: continue target, = dottedname attr: Optional[Documentable] = current.contents.get(target) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 8fe9b06ef..69aaf8228 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -54,6 +54,8 @@ def annotation_from_attrib( return None class ModuleVisitor(DataclassLikeVisitor): + + DATACLASS_LIKE_KIND = 'attrs class' def visit_ClassDef(self, node:ast.ClassDef) -> None: """ @@ -62,7 +64,7 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: super().visit_ClassDef(node) cls = self.visitor.builder._stack[-1].contents.get(node.name) - if not isinstance(cls, AttrsClass) or not cls.isDataclassLike: + if not isinstance(cls, AttrsClass) or not cls.dataclassLike: return mod = cls.module try: @@ -87,8 +89,10 @@ def transformClassVar(self, cls: model.Class, if annotation is None and value is not None: attr.annotation = annotation_from_attrib(value, cls) - def isDataclassLike(self, cls:ast.ClassDef, mod:model.Module) -> bool: - return any(is_attrs_deco(dec, mod) for dec in cls.decorator_list) + def isDataclassLike(self, cls:ast.ClassDef, mod:model.Module) -> Optional[object]: + if any(is_attrs_deco(dec, mod) for dec in cls.decorator_list): + return self.DATACLASS_LIKE_KIND + return None class AttrsClass(DataclasLikeClass, model.Class): auto_attribs: bool = False From f7220e96c25d2f036042e1f7b32955073cdefd3f Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 7 Jul 2023 10:59:25 -0400 Subject: [PATCH 24/48] Adjust postProcess --- pydoctor/extensions/attrs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index da606b0ef..9563fa31b 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -294,7 +294,7 @@ def postProcess(system:model.System) -> None: # by default attr.s() overrides any defined __init__ mehtod, whereas dataclasses. # TODO: but if auto_detect=True, we need to check if __init__ already exists, otherwise it does not replace it. # NOTE: But attr.define() use auto_detect=True by default! this is getting complicated... - if cls.isDataclassLike and cls.attrs_init: + if cls.dataclassLike == ModuleVisitor.DATACLASS_LIKE_KIND and cls.attrs_init: func = system.Function(system, '__init__', cls) system.addObject(func) # init Function attributes that otherwise would be undefined :/ From def0ee0f8f24a133db800f071a75b19cbba81193 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 7 Jul 2023 12:22:52 -0400 Subject: [PATCH 25/48] refarctors and add test --- pydoctor/extensions/attrs.py | 84 +++++++++++++++++------------------- pydoctor/test/test_attrs.py | 35 ++++++++++++--- 2 files changed, 70 insertions(+), 49 deletions(-) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 9563fa31b..80d7336a4 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -6,7 +6,7 @@ import ast import inspect -from typing import Dict, List, Optional, cast +from typing import Dict, List, Optional, TypedDict, cast from pydoctor import astbuilder, model, astutils, extensions from pydoctor.extensions._dataclass_like import DataclasLikeClass, DataclassLikeVisitor @@ -93,7 +93,7 @@ def _annotation_from_converter( elif isinstance(r, model.Function): args = dict(r.annotations) else: - return None + return astutils.dottedname2node(['object']) args.pop('return', None) if len(args)==1: return args.popitem()[1] @@ -143,17 +143,17 @@ def default_from_attrib(args:inspect.BoundArguments, ctx: model.Documentable) -> factory = get_factory(d, ctx) if factory: if astutils.node2dottedname(factory): - return ast.Call(func=factory, args=[], keywords=[], lineno=d.lineno, col_offset=d.col_offset) + return ast.Call(func=factory, args=[], keywords=[], lineno=d.lineno) else: - return ast.Constant(value=..., lineno=d.lineno, col_offset=d.col_offset) + return ast.Constant(value=..., lineno=d.lineno) return d elif isinstance(f, ast.expr): if astutils.node2dottedname(f): # If a simple factory is defined, the default value is a call to this function - return ast.Call(func=f, args=[], keywords=[], lineno=f.lineno, col_offset=f.col_offset) + return ast.Call(func=f, args=[], keywords=[], lineno=f.lineno) else: # Else we can't figure it out - return ast.Constant(value=..., lineno=f.lineno, col_offset=f.col_offset) + return ast.Constant(value=..., lineno=f.lineno) else: return None @@ -179,15 +179,12 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: attrs_args = astutils.safe_bind_args(attrs_decorator_signature, attrs_deco, mod) if attrs_args: - attrs_args_value = {name: astutils.get_literal_arg(attrs_args, name, default, + cls.attrs_options.update({name: astutils.get_literal_arg(attrs_args, name, default, typecheck, attrs_deco.lineno, mod ) for name, default, typecheck in (('auto_attribs', False, bool), ('init', True, bool), - ('kw_only', False, bool),)} - cls.attrs_auto_attribs = attrs_args_value['auto_attribs'] - cls.attrs_init = attrs_args_value['init'] - cls.attrs_kw_only = attrs_args_value['kw_only'] + ('kw_only', False, bool),)}) def transformClassVar(self, cls: model.Class, attr: model.Attribute, @@ -195,7 +192,7 @@ def transformClassVar(self, cls: model.Class, value:Optional[ast.expr]) -> None: assert isinstance(cls, AttrsClass) is_attrs_attrib = is_attrib(value, cls) - is_attrs_auto_attrib = cls.attrs_auto_attribs and not is_attrs_attrib and annotation is not None + is_attrs_auto_attrib = cls.attrs_options['auto_attribs'] and not is_attrs_attrib and annotation is not None attrib_args = None attrib_args_value = {} @@ -215,27 +212,25 @@ def transformClassVar(self, cls: model.Class, ('kw_only', False, bool),)} # Handle the auto-creation of the __init__ method. - if cls.attrs_init: + if cls.attrs_options['init']: if is_attrs_auto_attrib or (attrib_args and attrib_args_value['init']): kind:inspect._ParameterKind = inspect.Parameter.POSITIONAL_OR_KEYWORD - if cls.attrs_kw_only or (attrib_args and attrib_args_value['kw_only']): + if cls.attrs_options['kw_only'] or (attrib_args and attrib_args_value['kw_only']): kind = inspect.Parameter.KEYWORD_ONLY - attrs_default:Optional[ast.expr] = ast.Constant(value=...) + attrs_default:Optional[ast.expr] = ast.Constant(value=..., lineno=attr.linenumber) if is_attrs_auto_attrib: - attrs_default = value - factory = get_factory(attrs_default, cls) + factory = get_factory(value, cls) if factory: if astutils.node2dottedname(factory): - attrs_default = ast.Call(func=factory, args=[], keywords=[], - lineno=factory.lineno, col_offset=factory.col_offset) - else: - # Factory is not a default value stricly speaking, - # so we give up on trying to figure it out. - attrs_default = ast.Constant(value=..., lineno=factory.lineno, - col_offset=factory.col_offset) + attrs_default = ast.Call(func=factory, args=[], keywords=[], lineno=factory.lineno) + + # else, the factory is not a simple function/class name, + # so we give up on trying to figure it out. + else: + attrs_default = value elif attrib_args: attrs_default = default_from_attrib(attrib_args, cls) @@ -263,29 +258,30 @@ def isDataclassLike(self, cls:ast.ClassDef, mod:model.Module) -> Optional[object return self.DATACLASS_LIKE_KIND return None +class AttrsOptions(TypedDict): + auto_attribs: bool + """ + L{True} if this class uses the C{auto_attribs} feature of the L{attrs} + library to automatically convert annotated fields into attributes. + """ + + kw_only: bool + """ + C{True} is this class uses C{kw_only} feature of L{attrs } library. + """ + + init: bool + """ + False if L{attrs } is not generating an __init__ method for this class. + """ + class AttrsClass(DataclasLikeClass, model.Class): def setup(self) -> None: super().setup() - - self.attrs_auto_attribs: bool = False - """ - L{True} if this class uses the C{auto_attribs} feature of the L{attrs} - library to automatically convert annotated fields into attributes. - """ - - self.attrs_kw_only: bool = False - """ - C{True} is this class uses C{kw_only} feature of L{attrs } library. - """ - - self.attrs_init: bool = True - """ - False if L{attrs } is not generating an __init__ method for this class. - """ - # since the signatures doesnt include type annotations, we track them in a separate attribute. - self.attrs_constructor_parameters:List[inspect.Parameter] = [] - self.attrs_constructor_parameters.append(inspect.Parameter('self', inspect.Parameter.POSITIONAL_OR_KEYWORD,)) + self.attrs_options:AttrsOptions = {'init':True, 'auto_attribs':False, 'kw_only':False} + self.attrs_constructor_parameters:List[inspect.Parameter] = [ + inspect.Parameter('self', inspect.Parameter.POSITIONAL_OR_KEYWORD,)] self.attrs_constructor_annotations: Dict[str, Optional[ast.expr]] = {'self': None} def postProcess(system:model.System) -> None: @@ -294,7 +290,7 @@ def postProcess(system:model.System) -> None: # by default attr.s() overrides any defined __init__ mehtod, whereas dataclasses. # TODO: but if auto_detect=True, we need to check if __init__ already exists, otherwise it does not replace it. # NOTE: But attr.define() use auto_detect=True by default! this is getting complicated... - if cls.dataclassLike == ModuleVisitor.DATACLASS_LIKE_KIND and cls.attrs_init: + if cls.dataclassLike == ModuleVisitor.DATACLASS_LIKE_KIND and cls.attrs_options['init']: func = system.Function(system, '__init__', cls) system.addObject(func) # init Function attributes that otherwise would be undefined :/ diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 4b5e9b937..b0a7e067a 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -92,7 +92,7 @@ class C: ''', modname='test', systemcls=systemcls) C = mod.contents['C'] assert isinstance(C, attrs.AttrsClass) - assert C.attrs_auto_attribs == True + assert C.attrs_options['auto_attribs'] == True assert C.contents['a'].kind is model.DocumentableKind.INSTANCE_VARIABLE assert C.contents['b'].kind is model.DocumentableKind.INSTANCE_VARIABLE assert C.contents['c'].kind is model.DocumentableKind.CLASS_VARIABLE @@ -147,10 +147,10 @@ class D(C): assert capsys.readouterr().out == '' C = mod.contents['C'] assert isinstance(C, attrs.AttrsClass) - assert C.attrs_init == True + assert C.attrs_options['init'] == True D = mod.contents['D'] assert isinstance(D, attrs.AttrsClass) - assert D.attrs_init == False + assert D.attrs_options['init'] == False assert isinstance(C, model.Class) constructor = C.contents['__init__'] @@ -188,8 +188,8 @@ class C: mod = fromText(src, systemcls=systemcls) C = mod.contents['C'] assert isinstance(C, attrs.AttrsClass) - assert C.attrs_kw_only==True - assert C.attrs_init==True + assert C.attrs_options['kw_only']==True + assert C.attrs_options['init']==True constructor = C.contents['__init__'] assert isinstance(constructor, model.Function) assert flatten_text(pages.format_signature(constructor)) == '(self, *, a, b: str)' @@ -344,3 +344,28 @@ class MyClass: constructor = MyClass.contents['__init__'] assert isinstance(constructor, model.Function) assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, c: float, *, b: str)' + +@attrs_systemcls_param +def test_converter_init_annotation(systemcls) -> None: + src = '''\ + import attr + + class Stuff: + ... + + def convert_to_upper(value: object) -> str: + return str(value).upper() + + @attr.s + class MyClass: + name: str = attr.ib(converter=convert_to_upper) + st:Stuff = attr.ib(converter=Stuff) + age:int = attr.ib(converter=int) + ''' + + mod = fromText(src) + MyClass = mod.contents['MyClass'] + assert isinstance(MyClass, model.Class) + constructor = MyClass.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self, name: object, st: Stuff, age: object)' From 5a1939c00dd39494f6ee79b8828b94548dff5073 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 7 Jul 2023 16:28:21 -0400 Subject: [PATCH 26/48] Add support for auto_detect parameter --- pydoctor/astutils.py | 13 ++-- pydoctor/extensions/attrs.py | 135 +++++++++++++++++++---------------- pydoctor/test/test_attrs.py | 84 ++++++++++++++++++++-- 3 files changed, 161 insertions(+), 71 deletions(-) diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 504544dea..9c5d46ceb 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -15,9 +15,9 @@ if TYPE_CHECKING: from pydoctor import model - from typing import Protocol, Literal + from typing import Protocol, Literal, TypeGuard else: - Protocol = Literal = object + Protocol = Literal = TypeGuard = object # AST visitors @@ -455,7 +455,8 @@ def safe_bind_args(sig:Signature, call: ast.AST, ctx: 'model.Module') -> Optiona class _V(enum.Enum): NoValue = enum.auto() _T = TypeVar('_T', bound=object) -def _get_literal_arg(args:BoundArguments, name:str, typecheck:Type[_T]) -> Union['Literal[_V.NoValue]', _T]: +def _get_literal_arg(args:BoundArguments, name:str, + typecheck:'type[_T]|tuple[type[_T],...]') -> Union['Literal[_V.NoValue]', _T]: """ Helper function for L{get_literal_arg}. @@ -475,15 +476,17 @@ def _get_literal_arg(args:BoundArguments, name:str, typecheck:Type[_T]) -> Union raise ValueError(message) if not isinstance(value, typecheck): + expected_type = " or ".join(repr(t.__name__) for t in (typecheck if isinstance(typecheck, tuple) else (typecheck,))) message = (f'Value for {name!r} argument ' - f'has type "{type(value).__name__}", expected {typecheck.__name__!r}' + f'has type "{type(value).__name__}", expected {expected_type}' ).replace("'", '"') raise ValueError(message) return value def get_literal_arg(args:BoundArguments, name:str, default:_T, - typecheck:Type[_T], lineno:int, module: 'model.Module') -> _T: + typecheck:'type[_T]|tuple[type[_T],...]', + lineno:int, module: 'model.Module') -> _T: """ Retreive the literal value of an argument from the L{BoundArguments}. Only works with purely literal values (no C{Name} or C{Attribute}). diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 80d7336a4..b7534f950 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -168,7 +168,8 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: super().visit_ClassDef(node) cls = self.visitor.builder._stack[-1].contents.get(node.name) - if not isinstance(cls, AttrsClass) or not cls.dataclassLike: + if not isinstance(cls, AttrsClass) or cls.dataclassLike != self.DATACLASS_LIKE_KIND: + # not an attrs class return mod = cls.module try: @@ -183,8 +184,9 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: typecheck, attrs_deco.lineno, mod ) for name, default, typecheck in (('auto_attribs', False, bool), - ('init', True, bool), - ('kw_only', False, bool),)}) + ('init', None, (bool, type(None))), + ('kw_only', False, bool), + ('auto_detect', False, bool), )}) def transformClassVar(self, cls: model.Class, attr: model.Attribute, @@ -194,64 +196,66 @@ def transformClassVar(self, cls: model.Class, is_attrs_attrib = is_attrib(value, cls) is_attrs_auto_attrib = cls.attrs_options['auto_attribs'] and not is_attrs_attrib and annotation is not None + + if not (is_attrs_attrib or is_attrs_auto_attrib): + return + attrib_args = None attrib_args_value = {} - if is_attrs_attrib or is_attrs_auto_attrib: + + attr.kind = model.DocumentableKind.INSTANCE_VARIABLE + if value is not None: + attrib_args = astutils.safe_bind_args(attrib_signature, value, cls.module) + if attrib_args: + if annotation is None: + attr.annotation = annotation_from_attrib(attrib_args, cls) - attr.kind = model.DocumentableKind.INSTANCE_VARIABLE - if value is not None: - attrib_args = astutils.safe_bind_args(attrib_signature, value, cls.module) - if attrib_args: - if annotation is None: - attr.annotation = annotation_from_attrib(attrib_args, cls) - - attrib_args_value = {name: astutils.get_literal_arg(attrib_args, name, default, - typecheck, attr.linenumber, cls.module - ) for name, default, typecheck in - (('init', True, bool), - ('kw_only', False, bool),)} + attrib_args_value = {name: astutils.get_literal_arg(attrib_args, name, default, + typecheck, attr.linenumber, cls.module + ) for name, default, typecheck in + (('init', True, bool), + ('kw_only', False, bool),)} + + # Handle the auto-creation of the __init__ method. + if cls.attrs_options['init'] in (True, None) and is_attrs_auto_attrib or attrib_args_value.get('init'): + kind:inspect._ParameterKind = inspect.Parameter.POSITIONAL_OR_KEYWORD - # Handle the auto-creation of the __init__ method. - if cls.attrs_options['init']: - if is_attrs_auto_attrib or (attrib_args and attrib_args_value['init']): - kind:inspect._ParameterKind = inspect.Parameter.POSITIONAL_OR_KEYWORD - - if cls.attrs_options['kw_only'] or (attrib_args and attrib_args_value['kw_only']): - kind = inspect.Parameter.KEYWORD_ONLY - - attrs_default:Optional[ast.expr] = ast.Constant(value=..., lineno=attr.linenumber) - - if is_attrs_auto_attrib: - factory = get_factory(value, cls) - if factory: - if astutils.node2dottedname(factory): - attrs_default = ast.Call(func=factory, args=[], keywords=[], lineno=factory.lineno) - - # else, the factory is not a simple function/class name, - # so we give up on trying to figure it out. - else: - attrs_default = value - - elif attrib_args: - attrs_default = default_from_attrib(attrib_args, cls) - - # attrs strips the leading underscores from the parameter names, - # since there is not such thing as a private parameter. - init_param_name = attr.name.lstrip('_') - - if attrib_args: - constructor_annotation = cls.attrs_constructor_annotations[init_param_name] = \ - annotation_from_attrib(attrib_args, cls, for_constructor=True) or annotation - else: - constructor_annotation = cls.attrs_constructor_annotations[init_param_name] = annotation + if cls.attrs_options['kw_only'] or attrib_args_value.get('kw_only'): + kind = inspect.Parameter.KEYWORD_ONLY + + attrs_default:Optional[ast.expr] = ast.Constant(value=..., lineno=attr.linenumber) + + if is_attrs_auto_attrib: + factory = get_factory(value, cls) + if factory: + if astutils.node2dottedname(factory): + attrs_default = ast.Call(func=factory, args=[], keywords=[], lineno=factory.lineno) - cls.attrs_constructor_parameters.append( - inspect.Parameter( - init_param_name, kind=kind, - default=astbuilder._ValueFormatter(attrs_default, cls) - if attrs_default else inspect.Parameter.empty, - annotation=astbuilder._AnnotationValueFormatter(constructor_annotation, cls) - if constructor_annotation else inspect.Parameter.empty)) + # else, the factory is not a simple function/class name, + # so we give up on trying to figure it out. + else: + attrs_default = value + + elif attrib_args: + attrs_default = default_from_attrib(attrib_args, cls) + + # attrs strips the leading underscores from the parameter names, + # since there is not such thing as a private parameter. + init_param_name = attr.name.lstrip('_') + + if attrib_args: + constructor_annotation = annotation_from_attrib(attrib_args, cls, for_constructor=True) or annotation + else: + constructor_annotation = annotation + + cls.attrs_constructor_annotations[init_param_name] = constructor_annotation + cls.attrs_constructor_parameters.append( + inspect.Parameter( + init_param_name, kind=kind, + default=astbuilder._ValueFormatter(attrs_default, cls) + if attrs_default else inspect.Parameter.empty, + annotation=astbuilder._AnnotationValueFormatter(constructor_annotation, cls) + if constructor_annotation else inspect.Parameter.empty)) def isDataclassLike(self, cls:ast.ClassDef, mod:model.Module) -> Optional[object]: if any(is_attrs_deco(dec, mod) for dec in cls.decorator_list): @@ -270,16 +274,19 @@ class AttrsOptions(TypedDict): C{True} is this class uses C{kw_only} feature of L{attrs } library. """ - init: bool + init: Optional[bool] """ False if L{attrs } is not generating an __init__ method for this class. """ + auto_detect:bool + class AttrsClass(DataclasLikeClass, model.Class): def setup(self) -> None: super().setup() - self.attrs_options:AttrsOptions = {'init':True, 'auto_attribs':False, 'kw_only':False} + self.attrs_options:AttrsOptions = {'init':None, 'auto_attribs':False, + 'kw_only':False, 'auto_detect':False} self.attrs_constructor_parameters:List[inspect.Parameter] = [ inspect.Parameter('self', inspect.Parameter.POSITIONAL_OR_KEYWORD,)] self.attrs_constructor_annotations: Dict[str, Optional[ast.expr]] = {'self': None} @@ -290,13 +297,21 @@ def postProcess(system:model.System) -> None: # by default attr.s() overrides any defined __init__ mehtod, whereas dataclasses. # TODO: but if auto_detect=True, we need to check if __init__ already exists, otherwise it does not replace it. # NOTE: But attr.define() use auto_detect=True by default! this is getting complicated... - if cls.dataclassLike == ModuleVisitor.DATACLASS_LIKE_KIND and cls.attrs_options['init']: + if cls.dataclassLike == ModuleVisitor.DATACLASS_LIKE_KIND: + + if cls.attrs_options['init'] is False or \ + cls.attrs_options['init'] is None and \ + cls.attrs_options['auto_detect'] is True and \ + cls.contents.get('__init__'): + continue + func = system.Function(system, '__init__', cls) - system.addObject(func) # init Function attributes that otherwise would be undefined :/ + func.parentMod = cls.parentMod func.decorators = None func.is_async = False func.parentMod = cls.parentMod + system.addObject(func) func.setLineNumber(cls.linenumber) parameters = cls.attrs_constructor_parameters diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index b0a7e067a..d32105da7 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -147,10 +147,10 @@ class D(C): assert capsys.readouterr().out == '' C = mod.contents['C'] assert isinstance(C, attrs.AttrsClass) - assert C.attrs_options['init'] == True + assert C.attrs_options['init'] is None D = mod.contents['D'] assert isinstance(D, attrs.AttrsClass) - assert D.attrs_options['init'] == False + assert D.attrs_options['init'] is False assert isinstance(C, model.Class) constructor = C.contents['__init__'] @@ -188,8 +188,8 @@ class C: mod = fromText(src, systemcls=systemcls) C = mod.contents['C'] assert isinstance(C, attrs.AttrsClass) - assert C.attrs_options['kw_only']==True - assert C.attrs_options['init']==True + assert C.attrs_options['kw_only'] is True + assert C.attrs_options['init'] is None constructor = C.contents['__init__'] assert isinstance(constructor, model.Function) assert flatten_text(pages.format_signature(constructor)) == '(self, *, a, b: str)' @@ -346,7 +346,7 @@ class MyClass: assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, c: float, *, b: str)' @attrs_systemcls_param -def test_converter_init_annotation(systemcls) -> None: +def test_converter_init_annotation(systemcls:Type[model.System]) -> None: src = '''\ import attr @@ -363,9 +363,81 @@ class MyClass: age:int = attr.ib(converter=int) ''' - mod = fromText(src) + mod = fromText(src, systemcls=systemcls) MyClass = mod.contents['MyClass'] assert isinstance(MyClass, model.Class) constructor = MyClass.contents['__init__'] assert isinstance(constructor, model.Function) assert flatten_text(pages.format_signature(constructor)) == '(self, name: object, st: Stuff, age: object)' + +@attrs_systemcls_param +def test_auto_detect_init(systemcls:Type[model.System]) -> None: + src = '''\ + import attr + + @attr.s(auto_detect=True, auto_attribs=True) + class MyClass: + a: int + b: str + + def __init__(self): + self.a = 1 + self.b = 0 + + ''' + + mod = fromText(src, systemcls=systemcls) + MyClass = mod.contents['MyClass'] + assert isinstance(MyClass, model.Class) + constructor = MyClass.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self)' + +@attrs_systemcls_param +def test_auto_detect_is_False_init_overriden(systemcls:Type[model.System]) -> None: + src = '''\ + import attr + + @attr.s(auto_detect=False, auto_attribs=True) + class MyClass: + a: int + b: str + + def __init__(self): + self.a = 1 + self.b = 0 + + ''' + + mod = fromText(src, systemcls=systemcls) + MyClass = mod.contents['MyClass'] + assert isinstance(MyClass, model.Class) + constructor = MyClass.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str)' + +@attrs_systemcls_param +def test_auto_detect_is_True_init_is_True(systemcls:Type[model.System]) -> None: + # Passing ``True`` or ``False`` to *init*, *repr*, *eq*, *order*, + # *cmp*, or *hash* overrides whatever *auto_detect* would determine. + + src = '''\ + import attr + + @attr.s(auto_detect=True, auto_attribs=True, init=True) + class MyClass: + a: int + b: str + + def __init__(self): + self.a = 1 + self.b = 0 + + ''' + + mod = fromText(src, systemcls=systemcls) + MyClass = mod.contents['MyClass'] + assert isinstance(MyClass, model.Class) + constructor = MyClass.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str)' \ No newline at end of file From 411a584a45cd8512531a1b394b0e09793df24b63 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 7 Jul 2023 16:56:55 -0400 Subject: [PATCH 27/48] Support inherited constructor params --- pydoctor/extensions/attrs.py | 41 ++++++++++++++++++++++++++++++++---- pydoctor/test/test_attrs.py | 4 ++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index b7534f950..ce3498a4c 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -6,7 +6,7 @@ import ast import inspect -from typing import Dict, List, Optional, TypedDict, cast +from typing import Dict, List, Optional, Tuple, TypedDict, cast from pydoctor import astbuilder, model, astutils, extensions from pydoctor.extensions._dataclass_like import DataclasLikeClass, DataclassLikeVisitor @@ -291,6 +291,38 @@ def setup(self) -> None: inspect.Parameter('self', inspect.Parameter.POSITIONAL_OR_KEYWORD,)] self.attrs_constructor_annotations: Dict[str, Optional[ast.expr]] = {'self': None} +def collect_inherited_constructor_params(cls:AttrsClass) -> Tuple[List[inspect.Parameter], + Dict[str, Optional[ast.expr]]]: + # see https://github.com/python-attrs/attrs/pull/635/files + + base_attrs:List[inspect.Parameter] = [] + base_annotations:Dict[str, Optional[ast.expr]] = {} + own_attr_names = cls.attrs_constructor_annotations + + # Traverse the MRO and collect attributes. + for base_cls in reversed(cls.mro(include_external=False, include_self=False)): + assert isinstance(base_cls, AttrsClass) + for (name, ann),p in zip(base_cls.attrs_constructor_annotations.items(), + base_cls.attrs_constructor_parameters): + if name == 'self' or name in own_attr_names: + continue + + base_attrs.append(p) + base_annotations[name] = ann + + # For each name, only keep the freshest definition i.e. the furthest at the + # back. base_annotations is fine because it gets overwritten with every new + # instance. + filtered:List[inspect.Parameter] = [] + seen = set() + for a in reversed(base_attrs): + if a.name in seen: + continue + filtered.insert(0, a) + seen.add(a.name) + + return filtered, base_annotations + def postProcess(system:model.System) -> None: for cls in list(system.objectsOfType(AttrsClass)): @@ -314,8 +346,10 @@ def postProcess(system:model.System) -> None: system.addObject(func) func.setLineNumber(cls.linenumber) - parameters = cls.attrs_constructor_parameters - annotations = cls.attrs_constructor_annotations + # collect arguments from super classes attributes definitions. + inherited_params, inherited_annotations = collect_inherited_constructor_params(cls) + parameters = [cls.attrs_constructor_parameters[0], *inherited_params, *cls.attrs_constructor_parameters[1:]] + annotations = {**inherited_annotations, **cls.attrs_constructor_annotations} # Re-ordering kw_only arguments at the end of the list for param in tuple(parameters): @@ -328,7 +362,6 @@ def postProcess(system:model.System) -> None: func.annotations = annotations try: - # TODO: collect arguments from super classes attributes definitions. func.signature = inspect.Signature(parameters) except Exception as e: func.report(f'could not deduce attrs class __init__ signature: {e}') diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index d32105da7..86deda97e 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -285,7 +285,7 @@ class Derived(Base1, Base2): assert isinstance(Derived, model.Class) constructor = Derived.contents['__init__'] assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str, c: float)' + assert flatten_text(pages.format_signature(constructor)) == '(self, b: str, a: int, c: float)' # Test case for inheritance with overridden attributes: @attrs_systemcls_param @@ -307,7 +307,7 @@ class Derived(Base): assert isinstance(Derived, model.Class) constructor = Derived.contents['__init__'] assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str = "overridden", c: float = 3.14)' + assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str = \'overridden\', c: float = 3.14)' # Test case with attr.ib(init=False): @attrs_systemcls_param From 5c72350b329c0e2601ab7d639148c41fe995184e Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 7 Jul 2023 17:08:23 -0400 Subject: [PATCH 28/48] Fix presentation of constructors of attrs class --- pydoctor/epydoc2stan.py | 7 ++++--- pydoctor/extensions/attrs.py | 2 +- pydoctor/model.py | 3 --- pydoctor/templatewriter/pages/__init__.py | 6 ++++++ 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index fa34e94be..f2af6f341 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -1127,9 +1127,9 @@ def format_constructor_short_text(constructor: model.Function, forclass: model.C return f"{callable_name}({args})" -def populate_constructors_extra_info(cls:model.Class) -> None: +def get_constructors_extra(cls:model.Class) -> Optional[ParsedDocstring]: """ - Adds an extra information to be rendered based on Class constructors. + Get an extra docstring to represent Class constructors. """ from pydoctor.templatewriter import util constructors = cls.public_constructors @@ -1142,4 +1142,5 @@ def populate_constructors_extra_info(cls:model.Class) -> None: short_text = format_constructor_short_text(c, cls) extra_epytext += '`%s <%s>`' % (short_text, c.fullName()) - cls.extra_info.append(parse_docstring(cls, extra_epytext, cls, 'restructuredtext', section='constructor extra')) + return parse_docstring(cls, extra_epytext, cls, 'restructuredtext', section='constructor extra') + return None diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index ce3498a4c..2b4466f48 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -343,8 +343,8 @@ def postProcess(system:model.System) -> None: func.decorators = None func.is_async = False func.parentMod = cls.parentMod - system.addObject(func) func.setLineNumber(cls.linenumber) + system.addObject(func) # collect arguments from super classes attributes definitions. inherited_params, inherited_annotations = collect_inherited_constructor_params(cls) diff --git a/pydoctor/model.py b/pydoctor/model.py index 0ebafbbc3..250bacd7c 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -674,9 +674,6 @@ def _init_constructors(self) -> None: if return_ann == self.fullName() or \ return_ann in ('typing.Self', 'typing_extensions.Self'): self.constructors.append(fun) - - from pydoctor import epydoc2stan - epydoc2stan.populate_constructors_extra_info(self) @overload def mro(self, include_external:'Literal[True]', include_self:bool=True) -> Sequence[Union['Class', str]]:... diff --git a/pydoctor/templatewriter/pages/__init__.py b/pydoctor/templatewriter/pages/__init__.py index 272239605..18351beac 100644 --- a/pydoctor/templatewriter/pages/__init__.py +++ b/pydoctor/templatewriter/pages/__init__.py @@ -486,6 +486,12 @@ def extras(self) -> List["Flattenable"]: if p is not None: r.append(tags.p(p)) + constructor = epydoc2stan.get_constructors_extra(self.ob) + if constructor: + r.append(epydoc2stan.unwrap_docstring_stan( + epydoc2stan.safe_to_stan(constructor, self.ob.docstring_linker, self.ob, + fallback = lambda _,__,___:epydoc2stan.BROKEN, section='extra'))) + r.extend(super().extras()) return r From 37206046946aa120849a7280379fe2667392975b Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 7 Jul 2023 17:43:00 -0400 Subject: [PATCH 29/48] Fix order of arguments in constructor short text --- pydoctor/extensions/attrs.py | 11 ++++++----- pydoctor/test/test_attrs.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 2b4466f48..07c9fcc96 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -302,13 +302,12 @@ def collect_inherited_constructor_params(cls:AttrsClass) -> Tuple[List[inspect.P # Traverse the MRO and collect attributes. for base_cls in reversed(cls.mro(include_external=False, include_self=False)): assert isinstance(base_cls, AttrsClass) - for (name, ann),p in zip(base_cls.attrs_constructor_annotations.items(), - base_cls.attrs_constructor_parameters): - if name == 'self' or name in own_attr_names: + for p in base_cls.attrs_constructor_parameters[1:]: + if p.name in own_attr_names: continue base_attrs.append(p) - base_annotations[name] = ann + base_annotations[p.name] = base_cls.attrs_constructor_annotations[p.name] # For each name, only keep the freshest definition i.e. the furthest at the # back. base_annotations is fine because it gets overwritten with every new @@ -349,7 +348,9 @@ def postProcess(system:model.System) -> None: # collect arguments from super classes attributes definitions. inherited_params, inherited_annotations = collect_inherited_constructor_params(cls) parameters = [cls.attrs_constructor_parameters[0], *inherited_params, *cls.attrs_constructor_parameters[1:]] - annotations = {**inherited_annotations, **cls.attrs_constructor_annotations} + annotations = {'self': None} + annotations.update(inherited_annotations) + annotations.update(cls.attrs_constructor_annotations) # Re-ordering kw_only arguments at the end of the list for param in tuple(parameters): diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 86deda97e..95eb4a049 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -261,6 +261,7 @@ class Derived(Base): assert isinstance(Derived, model.Class) constructor = Derived.contents['__init__'] assert isinstance(constructor, model.Function) + assert epydoc2stan.format_constructor_short_text(constructor, Derived) == 'Derived(a, b)' assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str)' # Test case for multiple inheritance: @@ -285,6 +286,7 @@ class Derived(Base1, Base2): assert isinstance(Derived, model.Class) constructor = Derived.contents['__init__'] assert isinstance(constructor, model.Function) + assert epydoc2stan.format_constructor_short_text(constructor, Derived) == 'Derived(b, a, c)' assert flatten_text(pages.format_signature(constructor)) == '(self, b: str, a: int, c: float)' # Test case for inheritance with overridden attributes: @@ -307,8 +309,35 @@ class Derived(Base): assert isinstance(Derived, model.Class) constructor = Derived.contents['__init__'] assert isinstance(constructor, model.Function) + assert epydoc2stan.format_constructor_short_text(constructor, Derived) == 'Derived(a, b, c)' assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str = \'overridden\', c: float = 3.14)' +@attrs_systemcls_param +def test_attrs_constructor_single_inheritance_traverse_subclasses(systemcls: Type[model.System]) -> None: + src = '''\ + import attr + @attr.s(auto_attribs=True) + class FieldDesc: + name: Optional[str] = None + type: Optional[Tag] = None + body: Optional[Tag] = None + + @attr.s(auto_attribs=True) + class _SignatureDesc(FieldDesc): + type_origin: Optional[object] = None + + @attr.s(auto_attribs=True) + class ReturnDesc(_SignatureDesc):... + ''' + + mod = fromText(src, systemcls=systemcls) + ReturnDesc = mod.contents['ReturnDesc'] + assert isinstance(ReturnDesc, model.Class) + constructor = ReturnDesc.contents['__init__'] + assert isinstance(constructor, model.Function) + assert epydoc2stan.format_constructor_short_text(constructor, ReturnDesc) == 'ReturnDesc(name, type, body, type_origin)' + assert flatten_text(pages.format_signature(constructor)) == '(self, name: Optional[str] = None, type: Optional[Tag] = None, body: Optional[Tag] = None, type_origin: Optional[object] = None)' + # Test case with attr.ib(init=False): @attrs_systemcls_param def test_attrs_constructor_attribute_init_False(systemcls: Type[model.System]) -> None: From 299cd587b880e90cc686029214d4207d940f4382 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 7 Jul 2023 21:02:39 -0400 Subject: [PATCH 30/48] Fix keyword only feature --- pydoctor/extensions/attrs.py | 17 ++-- pydoctor/test/test_attrs.py | 154 +++++++++++++++++------------------ 2 files changed, 83 insertions(+), 88 deletions(-) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 07c9fcc96..7c30f4459 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -5,6 +5,7 @@ import ast import inspect +import copy from typing import Dict, List, Optional, Tuple, TypedDict, cast @@ -317,7 +318,7 @@ def collect_inherited_constructor_params(cls:AttrsClass) -> Tuple[List[inspect.P for a in reversed(base_attrs): if a.name in seen: continue - filtered.insert(0, a) + filtered.insert(0, copy.copy(a)) seen.add(a.name) return filtered, base_annotations @@ -326,8 +327,7 @@ def postProcess(system:model.System) -> None: for cls in list(system.objectsOfType(AttrsClass)): # by default attr.s() overrides any defined __init__ mehtod, whereas dataclasses. - # TODO: but if auto_detect=True, we need to check if __init__ already exists, otherwise it does not replace it. - # NOTE: But attr.define() use auto_detect=True by default! this is getting complicated... + # TODO: attr.define() use auto_detect=True by default. if cls.dataclassLike == ModuleVisitor.DATACLASS_LIKE_KIND: if cls.attrs_options['init'] is False or \ @@ -345,12 +345,15 @@ def postProcess(system:model.System) -> None: func.setLineNumber(cls.linenumber) system.addObject(func) - # collect arguments from super classes attributes definitions. + # collect arguments from super classes attributes definitions. inherited_params, inherited_annotations = collect_inherited_constructor_params(cls) + # don't forget to set the KEYWORD_ONLY flag on inherited parameters + if cls.attrs_options['kw_only'] is True: + for p in inherited_params: + p._kind = inspect.Parameter.KEYWORD_ONLY + # make sure that self is kept first. parameters = [cls.attrs_constructor_parameters[0], *inherited_params, *cls.attrs_constructor_parameters[1:]] - annotations = {'self': None} - annotations.update(inherited_annotations) - annotations.update(cls.attrs_constructor_annotations) + annotations = {'self': None, **inherited_annotations, **cls.attrs_constructor_annotations} # Re-ordering kw_only arguments at the end of the list for param in tuple(parameters): diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 95eb4a049..3daae605e 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -1,4 +1,4 @@ -from typing import Type +from typing import Optional, Type from pydoctor import epydoc2stan, model from pydoctor.extensions import attrs @@ -15,6 +15,16 @@ AttrsSystem, # system with attrs extension only )) +def assert_constructor(cls:model.Documentable, sig:str, + shortsig:Optional[str]=None) -> None: + assert isinstance(cls, model.Class) + constructor = cls.contents['__init__'] + assert isinstance(constructor, model.Function) + assert flatten_text(pages.format_signature(constructor)) == sig + if shortsig: + assert epydoc2stan.format_constructor_short_text(constructor, forclass=cls) == shortsig + + @attrs_systemcls_param def test_attrs_attrib_type(systemcls: Type[model.System]) -> None: """An attr.ib's "type" or "default" argument is used as an alternative @@ -152,11 +162,7 @@ class D(C): assert isinstance(D, attrs.AttrsClass) assert D.attrs_options['init'] is False - assert isinstance(C, model.Class) - constructor = C.contents['__init__'] - assert isinstance(constructor, model.Function) - assert epydoc2stan.format_constructor_short_text(constructor, forclass=C) == 'C(c, x, b)' - assert flatten_text(pages.format_signature(constructor)) == '(self, c: int = 100, x: int = 1, b: int = 23)' + assert_constructor(C, '(self, c: int = 100, x: int = 1, b: int = 23)', 'C(c, x, b)') # Test case for auto_attribs with defaults @attrs_systemcls_param @@ -169,11 +175,7 @@ class C: b: str = "default" ''' mod = fromText(src, systemcls=systemcls) - C = mod.contents['C'] - assert isinstance(C, model.Class) - constructor = C.contents['__init__'] - assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str = \'default\')' + assert_constructor(mod.contents['C'], '(self, a: int, b: str = \'default\')') # Test case for kw_only @attrs_systemcls_param @@ -190,9 +192,7 @@ class C: assert isinstance(C, attrs.AttrsClass) assert C.attrs_options['kw_only'] is True assert C.attrs_options['init'] is None - constructor = C.contents['__init__'] - assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self, *, a, b: str)' + assert_constructor(C, '(self, *, a, b: str)') # Test case for default factory @attrs_systemcls_param @@ -206,12 +206,8 @@ class C: c: list = attr.ib(default=attr.Factory(list)) ''' mod = fromText(src, systemcls=systemcls) - C = mod.contents['C'] - assert isinstance(C, model.Class) - constructor = C.contents['__init__'] - assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self, a: list = list(), b: str = str(), c: list = list())' - + assert_constructor(mod.contents['C'], '(self, a: list = list(), b: str = str(), c: list = list())') + @attrs_systemcls_param def test_attrs_constructor_factory_no_annotations(systemcls: Type[model.System]) -> None: src = '''\ @@ -222,12 +218,7 @@ class C: b = attr.ib(default=attr.Factory(list)) ''' mod = fromText(src, systemcls=systemcls) - C = mod.contents['C'] - assert isinstance(C, model.Class) - constructor = C.contents['__init__'] - assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self, a: list = list(), b: list = list())' - + assert_constructor(mod.contents['C'], '(self, a: list = list(), b: list = list())') # Test case for init=False: @attrs_systemcls_param @@ -257,12 +248,7 @@ class Derived(Base): b: str ''' mod = fromText(src, systemcls=systemcls) - Derived = mod.contents['Derived'] - assert isinstance(Derived, model.Class) - constructor = Derived.contents['__init__'] - assert isinstance(constructor, model.Function) - assert epydoc2stan.format_constructor_short_text(constructor, Derived) == 'Derived(a, b)' - assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str)' + assert_constructor(mod.contents['Derived'], '(self, a: int, b: str)', 'Derived(a, b)') # Test case for multiple inheritance: @attrs_systemcls_param @@ -282,12 +268,7 @@ class Derived(Base1, Base2): c: float ''' mod = fromText(src, systemcls=systemcls) - Derived = mod.contents['Derived'] - assert isinstance(Derived, model.Class) - constructor = Derived.contents['__init__'] - assert isinstance(constructor, model.Function) - assert epydoc2stan.format_constructor_short_text(constructor, Derived) == 'Derived(b, a, c)' - assert flatten_text(pages.format_signature(constructor)) == '(self, b: str, a: int, c: float)' + assert_constructor(mod.contents['Derived'], '(self, b: str, a: int, c: float)', 'Derived(b, a, c)') # Test case for inheritance with overridden attributes: @attrs_systemcls_param @@ -305,12 +286,7 @@ class Derived(Base): c: float = 3.14 ''' mod = fromText(src, systemcls=systemcls) - Derived = mod.contents['Derived'] - assert isinstance(Derived, model.Class) - constructor = Derived.contents['__init__'] - assert isinstance(constructor, model.Function) - assert epydoc2stan.format_constructor_short_text(constructor, Derived) == 'Derived(a, b, c)' - assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str = \'overridden\', c: float = 3.14)' + assert_constructor(mod.contents['Derived'], '(self, a: int, b: str = \'overridden\', c: float = 3.14)', 'Derived(a, b, c)') @attrs_systemcls_param def test_attrs_constructor_single_inheritance_traverse_subclasses(systemcls: Type[model.System]) -> None: @@ -331,12 +307,9 @@ class ReturnDesc(_SignatureDesc):... ''' mod = fromText(src, systemcls=systemcls) - ReturnDesc = mod.contents['ReturnDesc'] - assert isinstance(ReturnDesc, model.Class) - constructor = ReturnDesc.contents['__init__'] - assert isinstance(constructor, model.Function) - assert epydoc2stan.format_constructor_short_text(constructor, ReturnDesc) == 'ReturnDesc(name, type, body, type_origin)' - assert flatten_text(pages.format_signature(constructor)) == '(self, name: Optional[str] = None, type: Optional[Tag] = None, body: Optional[Tag] = None, type_origin: Optional[object] = None)' + assert_constructor(mod.contents['ReturnDesc'], + '(self, name: Optional[str] = None, type: Optional[Tag] = None, body: Optional[Tag] = None, type_origin: Optional[object] = None)', + 'ReturnDesc(name, type, body, type_origin)') # Test case with attr.ib(init=False): @attrs_systemcls_param @@ -349,11 +322,7 @@ class MyClass: b: str = attr.ib(init=False) ''' mod = fromText(src, systemcls=systemcls) - MyClass = mod.contents['MyClass'] - assert isinstance(MyClass, model.Class) - constructor = MyClass.contents['__init__'] - assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self, a: int)' + assert_constructor(mod.contents['MyClass'], '(self, a: int)') # Test case with attr.ib(kw_only=True): @attrs_systemcls_param @@ -368,11 +337,7 @@ class MyClass: ''' mod = fromText(src, systemcls=systemcls) assert not capsys.readouterr().out - MyClass = mod.contents['MyClass'] - assert isinstance(MyClass, model.Class) - constructor = MyClass.contents['__init__'] - assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, c: float, *, b: str)' + assert_constructor(mod.contents['MyClass'], '(self, a: int, c: float, *, b: str)') @attrs_systemcls_param def test_converter_init_annotation(systemcls:Type[model.System]) -> None: @@ -393,11 +358,7 @@ class MyClass: ''' mod = fromText(src, systemcls=systemcls) - MyClass = mod.contents['MyClass'] - assert isinstance(MyClass, model.Class) - constructor = MyClass.contents['__init__'] - assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self, name: object, st: Stuff, age: object)' + assert_constructor(mod.contents['MyClass'], '(self, name: object, st: Stuff, age: object)') @attrs_systemcls_param def test_auto_detect_init(systemcls:Type[model.System]) -> None: @@ -416,12 +377,8 @@ def __init__(self): ''' mod = fromText(src, systemcls=systemcls) - MyClass = mod.contents['MyClass'] - assert isinstance(MyClass, model.Class) - constructor = MyClass.contents['__init__'] - assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self)' - + assert_constructor(mod.contents['MyClass'], '(self)') + @attrs_systemcls_param def test_auto_detect_is_False_init_overriden(systemcls:Type[model.System]) -> None: src = '''\ @@ -439,11 +396,7 @@ def __init__(self): ''' mod = fromText(src, systemcls=systemcls) - MyClass = mod.contents['MyClass'] - assert isinstance(MyClass, model.Class) - constructor = MyClass.contents['__init__'] - assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str)' + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str)') @attrs_systemcls_param def test_auto_detect_is_True_init_is_True(systemcls:Type[model.System]) -> None: @@ -465,8 +418,47 @@ def __init__(self): ''' mod = fromText(src, systemcls=systemcls) - MyClass = mod.contents['MyClass'] - assert isinstance(MyClass, model.Class) - constructor = MyClass.contents['__init__'] - assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == '(self, a: int, b: str)' \ No newline at end of file + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str)') + +@attrs_systemcls_param +def test_field_keyword_only_inherited_parameters(systemcls) -> None: + src = '''\ + import attr + @attr.s + class A: + a = attr.ib(default=0) + @attr.s + class B(A): + b = attr.ib(kw_only=True) + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['B'], '(self, a: int = 0, *, b)') + +@attrs_systemcls_param +def test_class_keyword_only_inherited_parameters(systemcls) -> None: + # see https://github.com/python-attrs/attrs/commit/123df6704176d1981cf0d8f15a5021f4e2ce01ed + src = '''\ + import attr + @attr.s + class A: + a = attr.ib(default=0) + @attr.s(kw_only=True) + class B(A): + b = attr.ib() + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['A'], '(self, a: int = 0)') + assert_constructor(mod.contents['B'], '(self, *, a: int = 0, b)', 'B(a, b)') + + src = '''\ + import attr + @attr.s(auto_attribs=True) + class A: + a:int + @attr.s(auto_attribs=True, kw_only=True) + class B(A): + b:int + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['A'], '(self, a: int)') + assert_constructor(mod.contents['B'], '(self, *, a: int, b: int)', 'B(a, b)') \ No newline at end of file From 7497b7a2f4ba8b62c0f01656273e86fb0af1a797 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sat, 8 Jul 2023 18:39:46 -0400 Subject: [PATCH 31/48] Add support for the new APIs of attrs, fixes #718 --- pydoctor/astutils.py | 38 +++++++++++++- pydoctor/extensions/attrs.py | 89 +++++++++++++++++++++++++------- pydoctor/test/test_astbuilder.py | 40 ++++++++++++++ pydoctor/test/test_attrs.py | 78 +++++++++++++++++++++++++++- 4 files changed, 222 insertions(+), 23 deletions(-) diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 9c5d46ceb..e6ffc37d5 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -7,7 +7,7 @@ import platform import sys from numbers import Number -from typing import Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union, Type, TypeVar +from typing import Any, Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union, Type, TypeVar, Generic from inspect import BoundArguments, Signature, Parameter import ast @@ -511,3 +511,39 @@ def get_literal_arg(args:BoundArguments, name:str, default:_T, return default else: return value + +_TC = TypeVar('_TC', bound=object) +_SCOPE_TYPES = (ast.SetComp, ast.DictComp, ast.ListComp, ast.GeneratorExp, + ast.Lambda, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + +class _Collector(ast.NodeVisitor, Generic[_TC]): + + def __init__(self, + typecheck:Union[Type[_TC], Tuple[Type[_TC],...]], + stop_typecheck: Union[Type[Any], Tuple[Type[Any],...]], + ): + self.collected:List[_TC] = [] + self.typecheck = typecheck + self.stop_typecheck = stop_typecheck + + def _collect(self, node:ast.AST) -> None: + if isinstance(node, self.typecheck): + self.collected.append(node) + + def generic_visit(self, node: ast.AST) -> Any: + self._collect(node) + if not isinstance(node, self.stop_typecheck): + return super().generic_visit(node) + +def _collect_nodes(node:ast.AST, typecheck:Union[Type[_TC], Tuple[Type[_TC],...]], + stop_typecheck:Union[Type[Any], Tuple[Type[Any],...]]=_SCOPE_TYPES) -> Sequence[_TC]: + visitor:_Collector[_TC] = _Collector(typecheck, stop_typecheck) + ast.NodeVisitor.generic_visit(visitor, node) + return visitor.collected + +def collect_assigns(node:ast.AST) -> Sequence[Union[ast.Assign, ast.AnnAssign]]: + """ + Returns a list of L{ast.Assign} or L{ast.AnnAssign} declared in the given scope. + It does not include assignments in nested scopes. + """ + return _collect_nodes(node, (ast.Assign, ast.AnnAssign)) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 7c30f4459..c99d74537 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -4,10 +4,11 @@ """ import ast +import enum import inspect import copy -from typing import Dict, List, Optional, Tuple, TypedDict, cast +from typing import Dict, List, Optional, Sequence, Tuple, TypedDict, Union from pydoctor import astbuilder, model, astutils, extensions from pydoctor.extensions._dataclass_like import DataclasLikeClass, DataclassLikeVisitor @@ -24,16 +25,34 @@ 'complex', 'list', 'tuple', 'set', 'dict', 'range')) -def is_attrs_deco(deco: ast.AST, module: model.Module) -> bool: +class AttrsDeco(enum.Enum): + CLASSIC = 1 + """ + attr.s like + """ + + NEW = 2 + """ + attrs.define like + """ + +def attrs_deco_kind(deco: ast.AST, module: model.Module) -> Optional[AttrsDeco]: if isinstance(deco, ast.Call): deco = deco.func - return astutils.node2fullname(deco, module) in ( - 'attr.s', 'attr.attrs', 'attr.attributes') + if astutils.node2fullname(deco, module) in ( + 'attr.s', 'attr.attrs', 'attr.attributes'): + return AttrsDeco.CLASSIC + elif astutils.node2fullname(deco, module) in ( + 'attr.mutable', 'attr.frozen', 'attr.define', + 'attrs.mutable', 'attrs.frozen', 'attrs.define', + ): + return AttrsDeco.NEW + return None def is_attrib(expr: Optional[ast.expr], ctx: model.Documentable) -> bool: """Does this expression return an C{attr.ib}?""" return isinstance(expr, ast.Call) and astutils.node2fullname(expr.func, ctx) in ( - 'attr.ib', 'attr.attrib', 'attr.attr' + 'attr.ib', 'attr.attrib', 'attr.attr', 'attrs.field', 'attr.field' ) def get_factory(expr: Optional[ast.expr], ctx: model.Documentable) -> Optional[ast.expr]: @@ -158,6 +177,17 @@ def default_from_attrib(args:inspect.BoundArguments, ctx: model.Documentable) -> else: return None +def collect_fields(node:ast.ClassDef, ctx:model.Documentable) -> Sequence[Union[ast.Assign, ast.AnnAssign]]: + # used for the auto detection of auto_attribs value in newer APIs. + def _f(assign): + if isinstance(assign, ast.AnnAssign) and \ + not astutils.is_using_typing_classvar(assign.annotation, ctx): + return True + if is_attrib(assign.value): + return True + return False + return list(filter(_f, astutils.collect_assigns(node))) + class ModuleVisitor(DataclassLikeVisitor): DATACLASS_LIKE_KIND = 'attrs class' @@ -175,19 +205,37 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: mod = cls.module try: attrs_deco = next(decnode for decnode in node.decorator_list - if is_attrs_deco(decnode, mod)) + if attrs_deco_kind(decnode, mod)) except StopIteration: return - + + kind = attrs_deco_kind(attrs_deco, mod) attrs_args = astutils.safe_bind_args(attrs_decorator_signature, attrs_deco, mod) + + # init attrs options based on arguments and whether the devs are using + # the newer version of the APIs if attrs_args: + attrs_options = {'auto_attribs': (False, bool), + 'init': (None, (bool, type(None))), + 'kw_only': (False, bool), + 'auto_detect': (False, bool), } + + if kind == AttrsDeco.NEW: + attrs_options['auto_attribs'] = (None, (bool, type(None))) + attrs_options['auto_detect'] = (True, bool) + cls.attrs_options.update({name: astutils.get_literal_arg(attrs_args, name, default, typecheck, attrs_deco.lineno, mod - ) for name, default, typecheck in - (('auto_attribs', False, bool), - ('init', None, (bool, type(None))), - ('kw_only', False, bool), - ('auto_detect', False, bool), )}) + ) for name, (default, typecheck) in + attrs_options.items()}) + elif kind == AttrsDeco.NEW: + cls.attrs_options['auto_attribs'] = None + cls.attrs_options['auto_detect'] = True + + if kind == AttrsDeco.NEW and cls.attrs_options['auto_attribs'] is None: + fields = collect_fields(node, cls) + # auto detect auto_attrib value + cls.attrs_options['auto_attribs'] = len(fields)>0 and not any(isinstance(a, ast.Assign) for a in fields) def transformClassVar(self, cls: model.Class, attr: model.Attribute, @@ -197,7 +245,6 @@ def transformClassVar(self, cls: model.Class, is_attrs_attrib = is_attrib(value, cls) is_attrs_auto_attrib = cls.attrs_options['auto_attribs'] and not is_attrs_attrib and annotation is not None - if not (is_attrs_attrib or is_attrs_auto_attrib): return @@ -245,7 +292,8 @@ def transformClassVar(self, cls: model.Class, init_param_name = attr.name.lstrip('_') if attrib_args: - constructor_annotation = annotation_from_attrib(attrib_args, cls, for_constructor=True) or annotation + constructor_annotation = annotation_from_attrib( + attrib_args, cls, for_constructor=True) or annotation else: constructor_annotation = annotation @@ -259,7 +307,7 @@ def transformClassVar(self, cls: model.Class, if constructor_annotation else inspect.Parameter.empty)) def isDataclassLike(self, cls:ast.ClassDef, mod:model.Module) -> Optional[object]: - if any(is_attrs_deco(dec, mod) for dec in cls.decorator_list): + if any(attrs_deco_kind(dec, mod) for dec in cls.decorator_list): return self.DATACLASS_LIKE_KIND return None @@ -298,13 +346,13 @@ def collect_inherited_constructor_params(cls:AttrsClass) -> Tuple[List[inspect.P base_attrs:List[inspect.Parameter] = [] base_annotations:Dict[str, Optional[ast.expr]] = {} - own_attr_names = cls.attrs_constructor_annotations + own_param_names = cls.attrs_constructor_annotations # Traverse the MRO and collect attributes. for base_cls in reversed(cls.mro(include_external=False, include_self=False)): assert isinstance(base_cls, AttrsClass) for p in base_cls.attrs_constructor_parameters[1:]: - if p.name in own_attr_names: + if p.name in own_param_names: continue base_attrs.append(p) @@ -327,7 +375,6 @@ def postProcess(system:model.System) -> None: for cls in list(system.objectsOfType(AttrsClass)): # by default attr.s() overrides any defined __init__ mehtod, whereas dataclasses. - # TODO: attr.define() use auto_detect=True by default. if cls.dataclassLike == ModuleVisitor.DATACLASS_LIKE_KIND: if cls.attrs_options['init'] is False or \ @@ -352,8 +399,10 @@ def postProcess(system:model.System) -> None: for p in inherited_params: p._kind = inspect.Parameter.KEYWORD_ONLY # make sure that self is kept first. - parameters = [cls.attrs_constructor_parameters[0], *inherited_params, *cls.attrs_constructor_parameters[1:]] - annotations = {'self': None, **inherited_annotations, **cls.attrs_constructor_annotations} + parameters = [cls.attrs_constructor_parameters[0], + *inherited_params, *cls.attrs_constructor_parameters[1:]] + annotations = {'self': None, **inherited_annotations, + **cls.attrs_constructor_annotations} # Re-ordering kw_only arguments at the end of the list for param in tuple(parameters): diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index 573ff7998..95c0d52c1 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -1,3 +1,4 @@ +from textwrap import dedent from typing import Optional, Tuple, Type, List, overload, cast import ast @@ -2396,3 +2397,42 @@ def __init__(self): mod = fromText(src, systemcls=systemcls) assert getConstructorsText(mod.contents['Animal']) == "Animal()" + +# test for astutils.collect_assigns +def test_astutils_collect_assigns() -> None: + mod = ast.parse(dedent('''\ + class C: + def __init__(self):... + var:int + foo = True + class F: + [a for a in []] + {n:stuff for (n,stuff) in ()} + second = 1 + l = lambda x:True + ''' + )) + + C = mod.body[0] + F = C.body[-1] + # no assignment in module + assert [n for n in astutils.collect_assigns(mod)] == [] + # found one class in module + assert [n.name for n in astutils._collect_nodes(mod, ast.ClassDef)] == ['C'] + + # two attribute assignment in C + assert [n.lineno for n in astutils.collect_assigns(C)] == [3,4] + # one function + assert [n.name for n in astutils._collect_nodes(C, ast.FunctionDef)] == ['__init__'] + # one class + assert [n.name for n in astutils._collect_nodes(C, ast.ClassDef)] == ['F'] + + # two assignments in F + assert [n.lineno for n in astutils.collect_assigns(F)] == [8,9] + + # two names in F (it does not recurse on nested scopes) + assert [n.lineno for n in astutils._collect_nodes(F, ast.Name)] == [8,9] + + # two comprehensions + assert [n.lineno for n in astutils._collect_nodes(F, (ast.ListComp, ast.DictComp))] == [6,7] + diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 3daae605e..aa1abcee4 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -17,7 +17,8 @@ def assert_constructor(cls:model.Documentable, sig:str, shortsig:Optional[str]=None) -> None: - assert isinstance(cls, model.Class) + assert isinstance(cls, attrs.AttrsClass) + assert cls.dataclassLike == attrs.ModuleVisitor.DATACLASS_LIKE_KIND constructor = cls.contents['__init__'] assert isinstance(constructor, model.Function) assert flatten_text(pages.format_signature(constructor)) == sig @@ -378,7 +379,26 @@ def __init__(self): mod = fromText(src, systemcls=systemcls) assert_constructor(mod.contents['MyClass'], '(self)') + +@attrs_systemcls_param +def test_auto_detect_init_new_APIs(systemcls:Type[model.System]) -> None: + src = '''\ + import attr + + @attr.define + class MyClass: + a: int + b: str + + def __init__(self): + self.a = 1 + self.b = 0 + + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['MyClass'], '(self)') + @attrs_systemcls_param def test_auto_detect_is_False_init_overriden(systemcls:Type[model.System]) -> None: src = '''\ @@ -461,4 +481,58 @@ class B(A): ''' mod = fromText(src, systemcls=systemcls) assert_constructor(mod.contents['A'], '(self, a: int)') - assert_constructor(mod.contents['B'], '(self, *, a: int, b: int)', 'B(a, b)') \ No newline at end of file + assert_constructor(mod.contents['B'], '(self, *, a: int, b: int)', 'B(a, b)') + +@attrs_systemcls_param +def test_attrs_new_APIs_autodetect_auto_attribs_is_True(systemcls) -> None: + src = '''\ + import attrs as attr + + @attr.define(auto_attribs=None) + class MyClass: + a: int + b: str = attr.field(default=attr.Factory(str)) + ''' + mod = fromText(src, systemcls=systemcls) + assert mod.contents['MyClass'].attrs_options['auto_attribs']==True + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') + + mod = fromText(src.replace('@attr.define(auto_attribs=None)', '@attr.define'), systemcls=systemcls) + assert mod.contents['MyClass'].attrs_options['auto_attribs']==True + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') + + mod = fromText(src.replace('@attr.define(auto_attribs=None)', '@attr.mutable'), systemcls=systemcls) + assert mod.contents['MyClass'].attrs_options['auto_attribs']==True + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') + + mod = fromText(src.replace('@attr.define', '@attr.mutable'), systemcls=systemcls) + assert mod.contents['MyClass'].attrs_options['auto_attribs']==True + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') + + mod = fromText(src.replace('@attr.define(auto_attribs=None)', '@attr.frozen'), systemcls=systemcls) + assert mod.contents['MyClass'].attrs_options['auto_attribs']==True + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') + + mod = fromText(src.replace('@attr.define', '@attr.frozen'), systemcls=systemcls) + assert mod.contents['MyClass'].attrs_options['auto_attribs']==True + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') + + # older namespace + + mod = fromText(src.replace('import attrs as attr', 'import attr'), systemcls=systemcls) + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') + + mod = fromText(src.replace('@attr.define(auto_attribs=None)', '@attr.define').replace('import attrs as attr', 'import attr'), systemcls=systemcls) + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') + + mod = fromText(src.replace('@attr.define(auto_attribs=None)', '@attr.mutable').replace('import attrs as attr', 'import attr'), systemcls=systemcls) + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') + + mod = fromText(src.replace('@attr.define', '@attr.mutable').replace('import attrs as attr', 'import attr'), systemcls=systemcls) + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') + + mod = fromText(src.replace('@attr.define(auto_attribs=None)', '@attr.frozen').replace('import attrs as attr', 'import attr'), systemcls=systemcls) + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') + + mod = fromText(src.replace('@attr.define', '@attr.frozen').replace('import attrs as attr', 'import attr'), systemcls=systemcls) + assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') From 6a33634234c73f33f09839f87eb2582df0455aaa Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sat, 8 Jul 2023 22:10:05 -0400 Subject: [PATCH 32/48] Fix/silent mypy warnings and other refactors --- pydoctor/astutils.py | 48 ++++++------- pydoctor/extensions/attrs.py | 120 +++++++++++++++++-------------- pydoctor/test/test_astbuilder.py | 12 ++-- pydoctor/test/test_attrs.py | 37 +++++++--- 4 files changed, 119 insertions(+), 98 deletions(-) diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index e6ffc37d5..246578c69 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -15,9 +15,9 @@ if TYPE_CHECKING: from pydoctor import model - from typing import Protocol, Literal, TypeGuard + from typing import Protocol, Literal else: - Protocol = Literal = TypeGuard = object + Protocol = Literal = object # AST visitors @@ -482,7 +482,7 @@ def _get_literal_arg(args:BoundArguments, name:str, ).replace("'", '"') raise ValueError(message) - return value + return value #type:ignore def get_literal_arg(args:BoundArguments, name:str, default:_T, typecheck:'type[_T]|tuple[type[_T],...]', @@ -512,32 +512,24 @@ def get_literal_arg(args:BoundArguments, name:str, default:_T, else: return value -_TC = TypeVar('_TC', bound=object) _SCOPE_TYPES = (ast.SetComp, ast.DictComp, ast.ListComp, ast.GeneratorExp, ast.Lambda, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) - -class _Collector(ast.NodeVisitor, Generic[_TC]): - - def __init__(self, - typecheck:Union[Type[_TC], Tuple[Type[_TC],...]], - stop_typecheck: Union[Type[Any], Tuple[Type[Any],...]], - ): - self.collected:List[_TC] = [] - self.typecheck = typecheck - self.stop_typecheck = stop_typecheck - - def _collect(self, node:ast.AST) -> None: - if isinstance(node, self.typecheck): - self.collected.append(node) - - def generic_visit(self, node: ast.AST) -> Any: - self._collect(node) - if not isinstance(node, self.stop_typecheck): - return super().generic_visit(node) - -def _collect_nodes(node:ast.AST, typecheck:Union[Type[_TC], Tuple[Type[_TC],...]], - stop_typecheck:Union[Type[Any], Tuple[Type[Any],...]]=_SCOPE_TYPES) -> Sequence[_TC]: - visitor:_Collector[_TC] = _Collector(typecheck, stop_typecheck) +_ClassInfo = Union[Type[Any], Tuple[Type[Any],...]] + +def _collect_nodes(node:ast.AST, typecheck:_ClassInfo, + stop_typecheck:_ClassInfo=_SCOPE_TYPES) -> Sequence[ast.AST]: + class _Collector(ast.NodeVisitor): + def __init__(self) -> None: + self.collected:List[ast.AST] = [] + def _collect(self, node:ast.AST) -> None: + if isinstance(node, typecheck): + self.collected.append(node) + def generic_visit(self, node: ast.AST) -> None: + self._collect(node) + if not isinstance(node, stop_typecheck): + super().generic_visit(node) + + visitor = _Collector() ast.NodeVisitor.generic_visit(visitor, node) return visitor.collected @@ -546,4 +538,4 @@ def collect_assigns(node:ast.AST) -> Sequence[Union[ast.Assign, ast.AnnAssign]]: Returns a list of L{ast.Assign} or L{ast.AnnAssign} declared in the given scope. It does not include assignments in nested scopes. """ - return _collect_nodes(node, (ast.Assign, ast.AnnAssign)) + return _collect_nodes(node, (ast.Assign, ast.AnnAssign)) #type:ignore diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index c99d74537..db3066618 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -8,7 +8,7 @@ import inspect import copy -from typing import Dict, List, Optional, Sequence, Tuple, TypedDict, Union +from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, TypedDict, Union from pydoctor import astbuilder, model, astutils, extensions from pydoctor.extensions._dataclass_like import DataclasLikeClass, DataclassLikeVisitor @@ -179,15 +179,19 @@ def default_from_attrib(args:inspect.BoundArguments, ctx: model.Documentable) -> def collect_fields(node:ast.ClassDef, ctx:model.Documentable) -> Sequence[Union[ast.Assign, ast.AnnAssign]]: # used for the auto detection of auto_attribs value in newer APIs. - def _f(assign): + def _f(assign:Union[ast.Assign, ast.AnnAssign]) -> bool: if isinstance(assign, ast.AnnAssign) and \ not astutils.is_using_typing_classvar(assign.annotation, ctx): return True - if is_attrib(assign.value): + if is_attrib(assign.value, ctx): return True return False return list(filter(_f, astutils.collect_assigns(node))) +_fallback_attrs_call = ast.Call(func=ast.Name(id='define', ctx=ast.Load()), + args=[], keywords=[], lineno=0,) +_nothing = object() + class ModuleVisitor(DataclassLikeVisitor): DATACLASS_LIKE_KIND = 'attrs class' @@ -209,33 +213,41 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: except StopIteration: return + # init the self argument + cls.attrs_constructor_parameters.append( + inspect.Parameter('self', + inspect.Parameter.POSITIONAL_OR_KEYWORD) + ) + cls.attrs_constructor_annotations['self'] = None + kind = attrs_deco_kind(attrs_deco, mod) attrs_args = astutils.safe_bind_args(attrs_decorator_signature, attrs_deco, mod) # init attrs options based on arguments and whether the devs are using # the newer version of the APIs - if attrs_args: - attrs_options = {'auto_attribs': (False, bool), - 'init': (None, (bool, type(None))), - 'kw_only': (False, bool), - 'auto_detect': (False, bool), } - - if kind == AttrsDeco.NEW: - attrs_options['auto_attribs'] = (None, (bool, type(None))) - attrs_options['auto_detect'] = (True, bool) - - cls.attrs_options.update({name: astutils.get_literal_arg(attrs_args, name, default, - typecheck, attrs_deco.lineno, mod - ) for name, (default, typecheck) in - attrs_options.items()}) - elif kind == AttrsDeco.NEW: - cls.attrs_options['auto_attribs'] = None - cls.attrs_options['auto_detect'] = True - - if kind == AttrsDeco.NEW and cls.attrs_options['auto_attribs'] is None: + attrs_param_spec: Dict[str, Tuple[object, Union[Type[Any], Tuple[Type[Any],...]]]] = \ + {'auto_attribs': (False, bool), + 'init': (None, (bool, type(None))), + 'kw_only': (False, bool), + 'auto_detect': (False, bool), } + + if kind == AttrsDeco.NEW: + attrs_param_spec['auto_attribs'] = (None, (bool, type(None))) + attrs_param_spec['auto_detect'] = (True, bool) + + if not attrs_args: + attrs_args = astutils.bind_args(attrs_decorator_signature, _fallback_attrs_call) + + cls.attrs_options.update({name: astutils.get_literal_arg(attrs_args, name, default, + typecheck, attrs_deco.lineno, mod + ) for name, (default, typecheck) in + attrs_param_spec.items()}) + + if kind is AttrsDeco.NEW and cls.attrs_options['auto_attribs'] is None: fields = collect_fields(node, cls) # auto detect auto_attrib value - cls.attrs_options['auto_attribs'] = len(fields)>0 and not any(isinstance(a, ast.Assign) for a in fields) + cls.attrs_options['auto_attribs'] = len(fields)>0 and \ + not any(isinstance(a, ast.Assign) for a in fields) def transformClassVar(self, cls: model.Class, attr: model.Attribute, @@ -243,7 +255,8 @@ def transformClassVar(self, cls: model.Class, value:Optional[ast.expr]) -> None: assert isinstance(cls, AttrsClass) is_attrs_attrib = is_attrib(value, cls) - is_attrs_auto_attrib = cls.attrs_options['auto_attribs'] and not is_attrs_attrib and annotation is not None + is_attrs_auto_attrib = cls.attrs_options.get('auto_attribs') and \ + not is_attrs_attrib and annotation is not None if not (is_attrs_attrib or is_attrs_auto_attrib): return @@ -265,10 +278,11 @@ def transformClassVar(self, cls: model.Class, ('kw_only', False, bool),)} # Handle the auto-creation of the __init__ method. - if cls.attrs_options['init'] in (True, None) and is_attrs_auto_attrib or attrib_args_value.get('init'): + if cls.attrs_options.get('init', _nothing) in (True, None) and \ + is_attrs_auto_attrib or attrib_args_value.get('init'): + kind:inspect._ParameterKind = inspect.Parameter.POSITIONAL_OR_KEYWORD - - if cls.attrs_options['kw_only'] or attrib_args_value.get('kw_only'): + if cls.attrs_options.get('kw_only') or attrib_args_value.get('kw_only'): kind = inspect.Parameter.KEYWORD_ONLY attrs_default:Optional[ast.expr] = ast.Constant(value=..., lineno=attr.linenumber) @@ -277,7 +291,8 @@ def transformClassVar(self, cls: model.Class, factory = get_factory(value, cls) if factory: if astutils.node2dottedname(factory): - attrs_default = ast.Call(func=factory, args=[], keywords=[], lineno=factory.lineno) + attrs_default = ast.Call(func=factory, args=[], keywords=[], + lineno=factory.lineno) # else, the factory is not a simple function/class name, # so we give up on trying to figure it out. @@ -311,34 +326,33 @@ def isDataclassLike(self, cls:ast.ClassDef, mod:model.Module) -> Optional[object return self.DATACLASS_LIKE_KIND return None -class AttrsOptions(TypedDict): - auto_attribs: bool - """ - L{True} if this class uses the C{auto_attribs} feature of the L{attrs} - library to automatically convert annotated fields into attributes. +class AttrsOptions(Dict[str, object]): """ + Dictionary that may contain the following keys: - kw_only: bool - """ - C{True} is this class uses C{kw_only} feature of L{attrs } library. - """ + - auto_attribs: bool|None - init: Optional[bool] - """ - False if L{attrs } is not generating an __init__ method for this class. - """ + L{True} if this class uses the C{auto_attribs} feature of the L{attrs} + library to automatically convert annotated fields into attributes. - auto_detect:bool + - kw_only: bool + + C{True} is this class uses C{kw_only} feature of L{attrs } library. + + - init: bool|None + + False if L{attrs } is not generating an __init__ method for this class. + + - auto_detect:bool + """ class AttrsClass(DataclasLikeClass, model.Class): def setup(self) -> None: super().setup() - self.attrs_options:AttrsOptions = {'init':None, 'auto_attribs':False, - 'kw_only':False, 'auto_detect':False} - self.attrs_constructor_parameters:List[inspect.Parameter] = [ - inspect.Parameter('self', inspect.Parameter.POSITIONAL_OR_KEYWORD,)] - self.attrs_constructor_annotations: Dict[str, Optional[ast.expr]] = {'self': None} + self.attrs_options = AttrsOptions() + self.attrs_constructor_parameters: List[inspect.Parameter] = [] + self.attrs_constructor_annotations: Dict[str, Optional[ast.expr]] = {} def collect_inherited_constructor_params(cls:AttrsClass) -> Tuple[List[inspect.Parameter], Dict[str, Optional[ast.expr]]]: @@ -377,9 +391,9 @@ def postProcess(system:model.System) -> None: # by default attr.s() overrides any defined __init__ mehtod, whereas dataclasses. if cls.dataclassLike == ModuleVisitor.DATACLASS_LIKE_KIND: - if cls.attrs_options['init'] is False or \ - cls.attrs_options['init'] is None and \ - cls.attrs_options['auto_detect'] is True and \ + if cls.attrs_options.get('init') is False or \ + cls.attrs_options.get('init', _nothing) is None and \ + cls.attrs_options.get('auto_detect') is True and \ cls.contents.get('__init__'): continue @@ -395,13 +409,13 @@ def postProcess(system:model.System) -> None: # collect arguments from super classes attributes definitions. inherited_params, inherited_annotations = collect_inherited_constructor_params(cls) # don't forget to set the KEYWORD_ONLY flag on inherited parameters - if cls.attrs_options['kw_only'] is True: + if cls.attrs_options.get('kw_only') is True: for p in inherited_params: - p._kind = inspect.Parameter.KEYWORD_ONLY + p._kind = inspect.Parameter.KEYWORD_ONLY #type:ignore[attr-defined] # make sure that self is kept first. parameters = [cls.attrs_constructor_parameters[0], *inherited_params, *cls.attrs_constructor_parameters[1:]] - annotations = {'self': None, **inherited_annotations, + annotations:Dict[str, Optional[ast.expr]] = {'self': None, **inherited_annotations, **cls.attrs_constructor_annotations} # Re-ordering kw_only arguments at the end of the list diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index 95c0d52c1..aafab15cb 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -2414,25 +2414,21 @@ class F: )) C = mod.body[0] - F = C.body[-1] + F = C.body[-1] # type:ignore # no assignment in module assert [n for n in astutils.collect_assigns(mod)] == [] # found one class in module - assert [n.name for n in astutils._collect_nodes(mod, ast.ClassDef)] == ['C'] - + assert [n.name for n in astutils._collect_nodes(mod, ast.ClassDef)] == ['C'] # type:ignore # two attribute assignment in C assert [n.lineno for n in astutils.collect_assigns(C)] == [3,4] # one function - assert [n.name for n in astutils._collect_nodes(C, ast.FunctionDef)] == ['__init__'] + assert [n.name for n in astutils._collect_nodes(C, ast.FunctionDef)] == ['__init__'] # type:ignore # one class - assert [n.name for n in astutils._collect_nodes(C, ast.ClassDef)] == ['F'] - + assert [n.name for n in astutils._collect_nodes(C, ast.ClassDef)] == ['F'] # type:ignore # two assignments in F assert [n.lineno for n in astutils.collect_assigns(F)] == [8,9] - # two names in F (it does not recurse on nested scopes) assert [n.lineno for n in astutils._collect_nodes(F, ast.Name)] == [8,9] - # two comprehensions assert [n.lineno for n in astutils._collect_nodes(F, (ast.ListComp, ast.DictComp))] == [6,7] diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index aa1abcee4..7ccb518d2 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -441,7 +441,7 @@ def __init__(self): assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str)') @attrs_systemcls_param -def test_field_keyword_only_inherited_parameters(systemcls) -> None: +def test_field_keyword_only_inherited_parameters(systemcls:Type[model.System]) -> None: src = '''\ import attr @attr.s @@ -454,8 +454,10 @@ class B(A): mod = fromText(src, systemcls=systemcls) assert_constructor(mod.contents['B'], '(self, a: int = 0, *, b)') + + @attrs_systemcls_param -def test_class_keyword_only_inherited_parameters(systemcls) -> None: +def test_class_keyword_only_inherited_parameters(systemcls:Type[model.System]) -> None: # see https://github.com/python-attrs/attrs/commit/123df6704176d1981cf0d8f15a5021f4e2ce01ed src = '''\ import attr @@ -484,7 +486,7 @@ class B(A): assert_constructor(mod.contents['B'], '(self, *, a: int, b: int)', 'B(a, b)') @attrs_systemcls_param -def test_attrs_new_APIs_autodetect_auto_attribs_is_True(systemcls) -> None: +def test_attrs_new_APIs_autodetect_auto_attribs_is_True(systemcls:Type[model.System]) -> None: src = '''\ import attrs as attr @@ -494,27 +496,27 @@ class MyClass: b: str = attr.field(default=attr.Factory(str)) ''' mod = fromText(src, systemcls=systemcls) - assert mod.contents['MyClass'].attrs_options['auto_attribs']==True + assert mod.contents['MyClass'].attrs_options['auto_attribs']==True #type:ignore assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') mod = fromText(src.replace('@attr.define(auto_attribs=None)', '@attr.define'), systemcls=systemcls) - assert mod.contents['MyClass'].attrs_options['auto_attribs']==True + assert mod.contents['MyClass'].attrs_options['auto_attribs']==True #type:ignore assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') mod = fromText(src.replace('@attr.define(auto_attribs=None)', '@attr.mutable'), systemcls=systemcls) - assert mod.contents['MyClass'].attrs_options['auto_attribs']==True + assert mod.contents['MyClass'].attrs_options['auto_attribs']==True #type:ignore assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') mod = fromText(src.replace('@attr.define', '@attr.mutable'), systemcls=systemcls) - assert mod.contents['MyClass'].attrs_options['auto_attribs']==True + assert mod.contents['MyClass'].attrs_options['auto_attribs']==True #type:ignore assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') mod = fromText(src.replace('@attr.define(auto_attribs=None)', '@attr.frozen'), systemcls=systemcls) - assert mod.contents['MyClass'].attrs_options['auto_attribs']==True + assert mod.contents['MyClass'].attrs_options['auto_attribs']==True #type:ignore assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') mod = fromText(src.replace('@attr.define', '@attr.frozen'), systemcls=systemcls) - assert mod.contents['MyClass'].attrs_options['auto_attribs']==True + assert mod.contents['MyClass'].attrs_options['auto_attribs']==True #type:ignore assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') # older namespace @@ -536,3 +538,20 @@ class MyClass: mod = fromText(src.replace('@attr.define', '@attr.frozen').replace('import attrs as attr', 'import attr'), systemcls=systemcls) assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') + +@attrs_systemcls_param +def test_attrs_duplicate_param(systemcls: Type[model.System]) -> None: + src = '''\ + import attr + @attr.s(auto_attribs=True) + class MyClass: + a: int + + if int('36'): + @attr.s(auto_attribs=True) + class MyClass: + a: int + + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['MyClass'], '(self, a: int)') \ No newline at end of file From 4ce9b97ca01366271cb939a931ad2cd67283d93a Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sat, 8 Jul 2023 23:55:52 -0400 Subject: [PATCH 33/48] Add a docstring to attrs generated __init__ methods --- pydoctor/epydoc/markup/__init__.py | 7 +++ pydoctor/extensions/attrs.py | 70 ++++++++++++++++++++++-------- 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/pydoctor/epydoc/markup/__init__.py b/pydoctor/epydoc/markup/__init__.py index 709ff2c2b..2768ba5ed 100644 --- a/pydoctor/epydoc/markup/__init__.py +++ b/pydoctor/epydoc/markup/__init__.py @@ -223,6 +223,13 @@ def get_summary(self) -> 'ParsedDocstring': self._summary = visitor.summary or epydoc2stan.ParsedStanOnly(tags.span(class_='undocumented')("No summary")) return self._summary + def concatenate(self, other:'ParsedDocstring') -> 'ParsedDocstring': + from pydoctor.epydoc.markup.restructuredtext import ParsedRstDocstring + other_node = other.to_node() + self_node = self.to_node() + doc = new_document('docstring') + set_node_attributes(doc, children=[*self_node.children, *other_node.children]) + return ParsedRstDocstring(doc, self.fields) ################################################## ## Fields diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index db3066618..7428fdfbc 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -8,12 +8,21 @@ import inspect import copy -from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, TypedDict, Union +from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, Union -from pydoctor import astbuilder, model, astutils, extensions +import attr +from docutils import nodes + +from pydoctor import astbuilder, model, astutils, extensions, epydoc2stan from pydoctor.extensions._dataclass_like import DataclasLikeClass, DataclassLikeVisitor +from pydoctor.epydoc.markup import ParsedDocstring, Field +from pydoctor.epydoc.markup.restructuredtext import ParsedRstDocstring +from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval + +from pydoctor.epydoc2stan import parse_docstring +from pydoctor.epydoc.docutils import new_document, set_node_attributes + -import attr attrs_decorator_signature = inspect.signature(attr.s) """Signature of the L{attr.s} class decorator.""" @@ -57,7 +66,7 @@ def is_attrib(expr: Optional[ast.expr], ctx: model.Documentable) -> bool: def get_factory(expr: Optional[ast.expr], ctx: model.Documentable) -> Optional[ast.expr]: """ - If this AST represent a call to L{attr.Factory}, returns the expression inside the factory call + If this AST represent a call to L{attrs.Factory}, returns the expression inside the factory call """ if isinstance(expr, ast.Call) and \ astutils.node2fullname(expr.func, ctx) in ('attrs.Factory', 'attr.Factory'): @@ -308,9 +317,9 @@ def transformClassVar(self, cls: model.Class, if attrib_args: constructor_annotation = annotation_from_attrib( - attrib_args, cls, for_constructor=True) or annotation + attrib_args, cls, for_constructor=True) or attr.annotation else: - constructor_annotation = annotation + constructor_annotation = attr.annotation cls.attrs_constructor_annotations[init_param_name] = constructor_annotation cls.attrs_constructor_parameters.append( @@ -330,20 +339,20 @@ class AttrsOptions(Dict[str, object]): """ Dictionary that may contain the following keys: - - auto_attribs: bool|None + - auto_attribs: bool|None - L{True} if this class uses the C{auto_attribs} feature of the L{attrs} - library to automatically convert annotated fields into attributes. + L{True} if this class uses the C{auto_attribs} feature of the L{attrs} + library to automatically convert annotated fields into attributes. - - kw_only: bool - - C{True} is this class uses C{kw_only} feature of L{attrs } library. + - kw_only: bool + + C{True} is this class uses C{kw_only} feature of L{attrs } library. - - init: bool|None - - False if L{attrs } is not generating an __init__ method for this class. + - init: bool|None + + False if L{attrs } is not generating an __init__ method for this class. - - auto_detect:bool + - auto_detect:bool """ class AttrsClass(DataclasLikeClass, model.Class): @@ -385,6 +394,30 @@ def collect_inherited_constructor_params(cls:AttrsClass) -> Tuple[List[inspect.P return filtered, base_annotations +def craft_constructor_docstring(cls:AttrsClass, constructor_signature:inspect.Signature) -> ParsedDocstring: + fields = [] + for param in constructor_signature.parameters.values(): + if param.name=='self': + continue + attr = cls.find(param.name) + if isinstance(attr, model.Attribute): + doc_has_info = False + if is_attrib(attr.value, cls): + parsed_doc = colorize_inline_pyval(attr.value) + doc_has_info = True + else: + parsed_doc = parse_docstring(cls, '', cls, markup='epytext', section='attrs') + epydoc2stan.ensure_parsed_docstring(attr) + if attr.parsed_docstring: + parsed_doc = parsed_doc.concatenate(attr.parsed_docstring) + doc_has_info = True + if doc_has_info: + fields.append(Field('param', param.name, parsed_doc, lineno=cls.linenumber)) + doc = parse_docstring(cls, 'U{attrs } generated method', + cls, markup='epytext', section='attrs') + doc.fields = fields + return doc + def postProcess(system:model.System) -> None: for cls in list(system.objectsOfType(AttrsClass)): @@ -414,7 +447,7 @@ def postProcess(system:model.System) -> None: p._kind = inspect.Parameter.KEYWORD_ONLY #type:ignore[attr-defined] # make sure that self is kept first. parameters = [cls.attrs_constructor_parameters[0], - *inherited_params, *cls.attrs_constructor_parameters[1:]] + *inherited_params, *cls.attrs_constructor_parameters[1:]] annotations:Dict[str, Optional[ast.expr]] = {'self': None, **inherited_annotations, **cls.attrs_constructor_annotations} @@ -436,7 +469,8 @@ def postProcess(system:model.System) -> None: func.annotations = {} else: cls.constructors.append(func) - + func.parsed_docstring = craft_constructor_docstring(cls, func.signature) + def setup_pydoctor_extension(r:extensions.ExtRegistrar) -> None: r.register_astbuilder_visitor(ModuleVisitor) r.register_mixin(AttrsClass) From 4016b4cead2b6175d19de2663b5c0b44b536207c Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 9 Jul 2023 00:21:55 -0400 Subject: [PATCH 34/48] Fix bugs regarding the rendering of constructors --- pydoctor/extensions/attrs.py | 8 ++-- pydoctor/model.py | 72 ++++++++++++++++++------------------ 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 7428fdfbc..e60de632c 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -409,8 +409,11 @@ def craft_constructor_docstring(cls:AttrsClass, constructor_signature:inspect.Si parsed_doc = parse_docstring(cls, '', cls, markup='epytext', section='attrs') epydoc2stan.ensure_parsed_docstring(attr) if attr.parsed_docstring: - parsed_doc = parsed_doc.concatenate(attr.parsed_docstring) - doc_has_info = True + try: + parsed_doc = parsed_doc.concatenate(attr.parsed_docstring) + doc_has_info = True + except: + pass if doc_has_info: fields.append(Field('param', param.name, parsed_doc, lineno=cls.linenumber)) doc = parse_docstring(cls, 'U{attrs } generated method', @@ -468,7 +471,6 @@ def postProcess(system:model.System) -> None: func.signature = inspect.Signature() func.annotations = {} else: - cls.constructors.append(func) func.parsed_docstring = craft_constructor_docstring(cls, func.signature) def setup_pydoctor_extension(r:extensions.ExtRegistrar) -> None: diff --git a/pydoctor/model.py b/pydoctor/model.py index 250bacd7c..10dd4eb8c 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -606,6 +606,41 @@ def _find_dunder_constructor(cls:'Class') -> Optional['Function']: return _init return None +def get_constructors(cls:'Class') -> Iterator['Function']: + """ + Look for python language powered constructors or classmethod constructors. + + A constructor MUST be a method accessible in the locals of the class. + """ + # Look for python language powered constructors. + # If __new__ is defined, then it takes precedence over __init__ + # Blind spot: we don't understand when a Class is using a metaclass that overrides __call__. + dunder_constructor = _find_dunder_constructor(cls) + if dunder_constructor: + yield dunder_constructor + + # Then look for staticmethod/classmethod constructors, + # This only happens at the local scope level (i.e not looking in super-classes). + for fun in cls.contents.values(): + if not isinstance(fun, Function): + continue + # Only static methods and class methods can be recognized as constructors + if not fun.kind in (DocumentableKind.STATIC_METHOD, DocumentableKind.CLASS_METHOD): + continue + # get return annotation, if it returns the same type as self, it's a constructor method. + if not 'return' in fun.annotations: + # we currently only support constructor detection trought explicit annotations. + continue + + # annotation should be resolved at the module scope + return_ann = astutils.node2fullname(fun.annotations['return'], cls.module) + + # pydoctor understand explicit annotation as well as the Self-Type. + if return_ann == cls.fullName() or \ + return_ann in ('typing.Self', 'typing_extensions.Self'): + yield fun + + class Class(CanContainImportsDocumentable): kind = DocumentableKind.CLASS parent: CanContainImportsDocumentable @@ -621,7 +656,6 @@ def setup(self) -> None: self.rawbases: Sequence[Tuple[str, ast.expr]] = [] self.raw_decorators: Sequence[ast.expr] = [] self.subclasses: List[Class] = [] - self.constructors: List[Function] = [] """ List of constructors. @@ -642,39 +676,6 @@ def _init_mro(self) -> None: self.report(str(e), 'mro') self._mro = list(self.allbases(True)) - def _init_constructors(self) -> None: - """ - Initiate the L{Class.constructors} list. A constructor MUST be a method accessible - in the locals of the class. - """ - # Look for python language powered constructors. - # If __new__ is defined, then it takes precedence over __init__ - # Blind spot: we don't understand when a Class is using a metaclass that overrides __call__. - dunder_constructor = _find_dunder_constructor(self) - if dunder_constructor: - self.constructors.append(dunder_constructor) - - # Then look for staticmethod/classmethod constructors, - # This only happens at the local scope level (i.e not looking in super-classes). - for fun in self.contents.values(): - if not isinstance(fun, Function): - continue - # Only static methods and class methods can be recognized as constructors - if not fun.kind in (DocumentableKind.STATIC_METHOD, DocumentableKind.CLASS_METHOD): - continue - # get return annotation, if it returns the same type as self, it's a constructor method. - if not 'return' in fun.annotations: - # we currently only support constructor detection trought explicit annotations. - continue - - # annotation should be resolved at the module scope - return_ann = astutils.node2fullname(fun.annotations['return'], self.module) - - # pydoctor understand explicit annotation as well as the Self-Type. - if return_ann == self.fullName() or \ - return_ann in ('typing.Self', 'typing_extensions.Self'): - self.constructors.append(fun) - @overload def mro(self, include_external:'Literal[True]', include_self:bool=True) -> Sequence[Union['Class', str]]:... @overload @@ -727,7 +728,7 @@ def public_constructors(self) -> Sequence['Function']: arguments or have a docstring. """ r = [] - for c in self.constructors: + for c in get_constructors(self): if not c.isVisible: continue args = list(c.annotations) @@ -1440,7 +1441,6 @@ def postProcess(self) -> None: for cls in self.objectsOfType(Class): cls._init_mro() - cls._init_constructors() for b in cls.baseobjects: if b is not None: From eb833416e33f9aa953ad03b97159ee035012ab5a Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 9 Jul 2023 11:13:14 -0400 Subject: [PATCH 35/48] docs --- pydoctor/epydoc/markup/__init__.py | 5 ++++- pydoctor/extensions/attrs.py | 24 +++++++++++------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/pydoctor/epydoc/markup/__init__.py b/pydoctor/epydoc/markup/__init__.py index 2768ba5ed..c79ad5418 100644 --- a/pydoctor/epydoc/markup/__init__.py +++ b/pydoctor/epydoc/markup/__init__.py @@ -223,7 +223,10 @@ def get_summary(self) -> 'ParsedDocstring': self._summary = visitor.summary or epydoc2stan.ParsedStanOnly(tags.span(class_='undocumented')("No summary")) return self._summary - def concatenate(self, other:'ParsedDocstring') -> 'ParsedDocstring': + def concat(self, other:'ParsedDocstring') -> 'ParsedDocstring': + """ + Returns a new docstring with the content of the given docstring appended. + """ from pydoctor.epydoc.markup.restructuredtext import ParsedRstDocstring other_node = other.to_node() self_node = self.to_node() diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index e60de632c..846625fbf 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -394,28 +394,26 @@ def collect_inherited_constructor_params(cls:AttrsClass) -> Tuple[List[inspect.P return filtered, base_annotations -def craft_constructor_docstring(cls:AttrsClass, constructor_signature:inspect.Signature) -> ParsedDocstring: +def attrs_constructor_docstring(cls:AttrsClass, constructor_signature:inspect.Signature) -> ParsedDocstring: + """ + Get a docstring for the attrs generated constructor method + """ fields = [] for param in constructor_signature.parameters.values(): if param.name=='self': continue attr = cls.find(param.name) if isinstance(attr, model.Attribute): - doc_has_info = False if is_attrib(attr.value, cls): - parsed_doc = colorize_inline_pyval(attr.value) - doc_has_info = True + field_doc = colorize_inline_pyval(attr.value) else: - parsed_doc = parse_docstring(cls, '', cls, markup='epytext', section='attrs') + field_doc = parse_docstring(cls, '', cls, markup='epytext', section='attrs') epydoc2stan.ensure_parsed_docstring(attr) if attr.parsed_docstring: - try: - parsed_doc = parsed_doc.concatenate(attr.parsed_docstring) - doc_has_info = True - except: - pass - if doc_has_info: - fields.append(Field('param', param.name, parsed_doc, lineno=cls.linenumber)) + field_doc = field_doc.concat(attr.parsed_docstring) + if field_doc.has_body: + fields.append(Field('param', param.name, field_doc, lineno=cls.linenumber)) + doc = parse_docstring(cls, 'U{attrs } generated method', cls, markup='epytext', section='attrs') doc.fields = fields @@ -471,7 +469,7 @@ def postProcess(system:model.System) -> None: func.signature = inspect.Signature() func.annotations = {} else: - func.parsed_docstring = craft_constructor_docstring(cls, func.signature) + func.parsed_docstring = attrs_constructor_docstring(cls, func.signature) def setup_pydoctor_extension(r:extensions.ExtRegistrar) -> None: r.register_astbuilder_visitor(ModuleVisitor) From 63edbedc61a0c01f13ba7f28b13da3e2aa842ede Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 9 Jul 2023 18:22:35 -0400 Subject: [PATCH 36/48] Add documentation section on improved attrs support --- docs/source/codedoc.rst | 42 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/source/codedoc.rst b/docs/source/codedoc.rst index 06fa4325d..f2a20e2eb 100644 --- a/docs/source/codedoc.rst +++ b/docs/source/codedoc.rst @@ -26,8 +26,7 @@ Pydoctor also supports *attribute docstrings*:: """This docstring describes a class variable.""" def __init__(self): - self.ivar = [] - """This docstring describes an instance variable.""" + self.ivar = [];"It can also be used inline." Attribute docstrings are not part of the Python language itself (`PEP 224 `_ was rejected), so these docstrings are not available at runtime. @@ -284,6 +283,45 @@ If you are using explicit ``attr.ib`` definitions instead of ``auto_attribs``, p list_of_numbers = attr.ib(factory=list) # type: List[int] """Multiple numbers.""" +Pydoctor look for ``attrs`` fields declarations and analyze the +arguments passed to ``attr.s`` and ``attr.ib`` in order to +precisely infer what's the signature of the constructor method:: + + from typing import List + import pathlib + import attr + + def convert_paths(p:List[str]) -> List[pathlib.Path]: + return [pathlib.Path(s) for s in p] + + @attr.s(auto_attribs=True) + class Base: + a: int + + @attr.s(auto_attribs=True, kw_only=True) + class SomeClass(Base): + a_number:int=42; "docstring of number A." + list_of_numbers:List[int] = attr.ib(factory=list); "List of ints" + converted_paths:List[pathlib.Path] = attr.ib(converter=convert_paths, factory=list); "Uses a converter" + +The constrcutor method will be documented as if it was explicitly defined, +with a docstring including documentation of each parameters and a note +saying the method is generated by attrs:: + + def __init__(self, *, a: int, a_number: int = 42, + list_of_numbers: list = list(), + converted_paths: List[str] = list()): + """ + attrs generated method + + @param a_number: docstring of number A. + @param list_of_numbers: C{attr.ib(factory=list)} + List of ints + @param converted_paths: C{attr.ib(converter=convert_paths, factory=list)} + Uses a converter + """ + +Pydoctor also supports the newer APIs (``attrs.define``/``attrs.field``). Private API ----------- From 1e82784fcdba0adc582f49ed7861c0cd185e4d4a Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 9 Jul 2023 18:22:41 -0400 Subject: [PATCH 37/48] Add tests --- pydoctor/test/test_astbuilder.py | 9 +++++++ pydoctor/test/test_attrs.py | 44 +++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index aafab15cb..3cf2bc4da 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -2398,6 +2398,15 @@ def __init__(self): mod = fromText(src, systemcls=systemcls) assert getConstructorsText(mod.contents['Animal']) == "Animal()" +@systemcls_param +def test_docstring_attribute_inline(systemcls:Type[model.System]) -> None: + src='''\ + class SomeClass(Base): + a_number:int=42; "docstring of number A." + ''' + mod = fromText(src, systemcls=systemcls) + assert mod.contents['SomeClass'].contents['a_number'].docstring=="docstring of number A." + # test for astutils.collect_assigns def test_astutils_collect_assigns() -> None: mod = ast.parse(dedent('''\ diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 7ccb518d2..2c91ddbfe 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -554,4 +554,46 @@ class MyClass: ''' mod = fromText(src, systemcls=systemcls) - assert_constructor(mod.contents['MyClass'], '(self, a: int)') \ No newline at end of file + assert_constructor(mod.contents['MyClass'], '(self, a: int)') + +@attrs_systemcls_param +def test_type_comment_wins_over_factory_annotation(systemcls: Type[model.System]) -> None: + src = '''\ + from typing import List + import attr + + @attr.s + class SomeClass: + + a_number = attr.ib(default=42) + """One number.""" + + list_of_numbers = attr.ib(factory=list) # type: List[int] + """Multiple numbers.""" + ''' + + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['SomeClass'], '(self, a_number: int = 42, list_of_numbers: List[int] = list())') + +@attrs_systemcls_param +def test_type_comment_wins_over_factory_annotation(systemcls: Type[model.System]) -> None: + src = '''\ + from typing import List + import pathlib + import attr + + def convert_paths(p:List[str]) -> List[pathlib.Path]: + return [pathlib.Path(s) for s in p] + + @attr.s(auto_attribs=True) + class Base: + a: int + + @attr.s(auto_attribs=True, kw_only=True) + class SomeClass(Base): + a_number:int=42 + list_of_numbers:List[int] = attr.ib(factory=list) + converted_paths:List[pathlib.Path] = attr.ib(converter=convert_paths, factory=list)''' + + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['SomeClass'], '(self, *, a: int, a_number: int = 42, list_of_numbers: list = list(), converted_paths: List[str] = list())') From bcce4f3d946f7b41fa4c0668cedb6ee76dc6e0b1 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 9 Jul 2023 20:32:32 -0400 Subject: [PATCH 38/48] Fix little issue of priorization of presented annotations --- docs/source/codedoc.rst | 2 +- pydoctor/epydoc/markup/plaintext.py | 2 +- pydoctor/extensions/attrs.py | 30 ++++++++-------- pydoctor/test/test_attrs.py | 53 +++++++++++++++++++++++++---- 4 files changed, 64 insertions(+), 23 deletions(-) diff --git a/docs/source/codedoc.rst b/docs/source/codedoc.rst index f2a20e2eb..9991c26e5 100644 --- a/docs/source/codedoc.rst +++ b/docs/source/codedoc.rst @@ -309,7 +309,7 @@ with a docstring including documentation of each parameters and a note saying the method is generated by attrs:: def __init__(self, *, a: int, a_number: int = 42, - list_of_numbers: list = list(), + list_of_numbers: List[int] = list(), converted_paths: List[str] = list()): """ attrs generated method diff --git a/pydoctor/epydoc/markup/plaintext.py b/pydoctor/epydoc/markup/plaintext.py index 22a438bb9..bdfaa7023 100644 --- a/pydoctor/epydoc/markup/plaintext.py +++ b/pydoctor/epydoc/markup/plaintext.py @@ -69,7 +69,7 @@ def to_node(self) -> nodes.document: paragraphs = [set_node_attributes(nodes.paragraph('',''), children=[ set_node_attributes(nodes.Text(p.strip('\n')), document=_document, lineno=0)], document=_document, lineno=0) - for p in self._text.split('\n\n')] + for p in self._text.split('\n\n') if p not in ('', '\n')] # assemble document _document = set_node_attributes(_document, diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 846625fbf..d054622ed 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -11,12 +11,11 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, Union import attr -from docutils import nodes from pydoctor import astbuilder, model, astutils, extensions, epydoc2stan from pydoctor.extensions._dataclass_like import DataclasLikeClass, DataclassLikeVisitor from pydoctor.epydoc.markup import ParsedDocstring, Field -from pydoctor.epydoc.markup.restructuredtext import ParsedRstDocstring +from pydoctor.epydoc.markup.plaintext import ParsedPlaintextDocstring from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval from pydoctor.epydoc2stan import parse_docstring @@ -151,17 +150,18 @@ def annotation_from_attrib( if typ is not None: return astutils.unstring_annotation(typ, ctx) - factory = args.arguments.get('factory') - if factory is not None: - return _annotation_from_factory(factory, ctx) - - default = args.arguments.get('default') - if default is not None: - factory = get_factory(default, ctx) + if not for_constructor: + factory = args.arguments.get('factory') if factory is not None: return _annotation_from_factory(factory, ctx) - else: - return astbuilder._infer_type(default) + + default = args.arguments.get('default') + if default is not None: + factory = get_factory(default, ctx) + if factory is not None: + return _annotation_from_factory(factory, ctx) + else: + return astbuilder._infer_type(default) return None @@ -277,7 +277,7 @@ def transformClassVar(self, cls: model.Class, if value is not None: attrib_args = astutils.safe_bind_args(attrib_signature, value, cls.module) if attrib_args: - if annotation is None: + if annotation is None and attr.annotation is None: attr.annotation = annotation_from_attrib(attrib_args, cls) attrib_args_value = {name: astutils.get_literal_arg(attrib_args, name, default, @@ -317,7 +317,9 @@ def transformClassVar(self, cls: model.Class, if attrib_args: constructor_annotation = annotation_from_attrib( - attrib_args, cls, for_constructor=True) or attr.annotation + attrib_args, cls, for_constructor=True) or \ + attr.annotation or annotation_from_attrib( + attrib_args, cls) else: constructor_annotation = attr.annotation @@ -407,7 +409,7 @@ def attrs_constructor_docstring(cls:AttrsClass, constructor_signature:inspect.Si if is_attrib(attr.value, cls): field_doc = colorize_inline_pyval(attr.value) else: - field_doc = parse_docstring(cls, '', cls, markup='epytext', section='attrs') + field_doc = ParsedPlaintextDocstring('') epydoc2stan.ensure_parsed_docstring(attr) if attr.parsed_docstring: field_doc = field_doc.concat(attr.parsed_docstring) diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 2c91ddbfe..6b5043479 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -1,8 +1,10 @@ +import re from typing import Optional, Type from pydoctor import epydoc2stan, model from pydoctor.extensions import attrs from pydoctor.stanutils import flatten_text +from pydoctor.node2stan import gettext from pydoctor.templatewriter import pages from pydoctor.test import CapSys @@ -202,12 +204,12 @@ def test_attrs_constructor_factory(systemcls: Type[model.System]) -> None: import attr @attr.s(auto_attribs=True) class C: - a: int = attr.ib(factory=list) + a: int = attr.ib(factory=lambda:False) b: str = attr.Factory(str) c: list = attr.ib(default=attr.Factory(list)) ''' mod = fromText(src, systemcls=systemcls) - assert_constructor(mod.contents['C'], '(self, a: list = list(), b: str = str(), c: list = list())') + assert_constructor(mod.contents['C'], '(self, a: int = ..., b: str = str(), c: list = list())') @attrs_systemcls_param def test_attrs_constructor_factory_no_annotations(systemcls: Type[model.System]) -> None: @@ -539,6 +541,21 @@ class MyClass: mod = fromText(src.replace('@attr.define', '@attr.frozen').replace('import attrs as attr', 'import attr'), systemcls=systemcls) assert_constructor(mod.contents['MyClass'], '(self, a: int, b: str = str())') +@attrs_systemcls_param +def test_attrs_new_APIs_autodetect_auto_attribs_is_False(systemcls:Type[model.System]) -> None: + src = '''\ + import attrs as attr + + @attr.define + class MyClass: + a: int + b = attr.field(factory=set) + c = 42 + ''' + mod = fromText(src, systemcls=systemcls) + assert mod.contents['MyClass'].attrs_options['auto_attribs']==False #type:ignore + assert_constructor(mod.contents['MyClass'], '(self, b: set = set())') + @attrs_systemcls_param def test_attrs_duplicate_param(systemcls: Type[model.System]) -> None: src = '''\ @@ -576,7 +593,7 @@ class SomeClass: assert_constructor(mod.contents['SomeClass'], '(self, a_number: int = 42, list_of_numbers: List[int] = list())') @attrs_systemcls_param -def test_type_comment_wins_over_factory_annotation(systemcls: Type[model.System]) -> None: +def test_docstring_generated(systemcls: Type[model.System]) -> None: src = '''\ from typing import List import pathlib @@ -591,9 +608,31 @@ class Base: @attr.s(auto_attribs=True, kw_only=True) class SomeClass(Base): - a_number:int=42 - list_of_numbers:List[int] = attr.ib(factory=list) - converted_paths:List[pathlib.Path] = attr.ib(converter=convert_paths, factory=list)''' + a_number:int=42; "docstring of number A" + list_of_numbers:List[int] = attr.ib(factory=list); "List of ints" + converted_paths:List[pathlib.Path] = attr.ib(converter=convert_paths, factory=list); "Uses a converter" + ''' + + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['SomeClass'], '(self, *, a: int, a_number: int = 42, list_of_numbers: List[int] = list(), converted_paths: List[str] = list())') + + __init__ = mod.contents['SomeClass'].contents['__init__'] + assert re.match( + r'''attrs generated method''', + ''.join(gettext(__init__.parsed_docstring.to_node()))) + assert len(__init__.parsed_docstring.fields)==3 + assert re.match( + r'''docstring of number A\sattr.ib\(factory=list\)List of ints\sattr.ib\(converter=convert_paths, factory=list\)Uses a converter''', + ' '.join(text for text in (''.join(gettext(f.body().to_node())) for f in __init__.parsed_docstring.fields)) + ) +@attrs_systemcls_param +def test_define_type_comment_not_auto_attribs(systemcls: Type[model.System]) -> None: + # this should be interpreted as using auto_attribs=False + src='''\ + import attr + @attr.define + class A: + a = 0 #type:int''' mod = fromText(src, systemcls=systemcls) - assert_constructor(mod.contents['SomeClass'], '(self, *, a: int, a_number: int = 42, list_of_numbers: list = list(), converted_paths: List[str] = list())') + assert_constructor(mod.contents['A'], '(self)') \ No newline at end of file From ad9bfef30d26ea9722101a3b452e401ebe2a40cc Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 14 Jul 2023 02:19:05 -0400 Subject: [PATCH 39/48] Fix mypy --- pydoctor/extensions/attrs.py | 2 +- pydoctor/templatewriter/__init__.py | 3 +-- pydoctor/test/test_attrs.py | 6 +++--- pydoctor/test/test_sphinx.py | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index d054622ed..edbd5ea26 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -407,7 +407,7 @@ def attrs_constructor_docstring(cls:AttrsClass, constructor_signature:inspect.Si attr = cls.find(param.name) if isinstance(attr, model.Attribute): if is_attrib(attr.value, cls): - field_doc = colorize_inline_pyval(attr.value) + field_doc: ParsedDocstring = colorize_inline_pyval(attr.value) else: field_doc = ParsedPlaintextDocstring('') epydoc2stan.ensure_parsed_docstring(attr) diff --git a/pydoctor/templatewriter/__init__.py b/pydoctor/templatewriter/__init__.py index ffaff07a1..e9ad6ddec 100644 --- a/pydoctor/templatewriter/__init__.py +++ b/pydoctor/templatewriter/__init__.py @@ -38,8 +38,7 @@ def parse_xml(text: str) -> minidom.Document: Create a L{minidom} representaton of the XML string. """ try: - # TODO: submit a PR to typeshed to add a return type for parseString() - return cast(minidom.Document, minidom.parseString(text)) + return minidom.parseString(text) except Exception as e: raise ValueError(f"Failed to parse template as XML: {e}") from e diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 6b5043479..85c1c3177 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -619,11 +619,11 @@ class SomeClass(Base): __init__ = mod.contents['SomeClass'].contents['__init__'] assert re.match( r'''attrs generated method''', - ''.join(gettext(__init__.parsed_docstring.to_node()))) - assert len(__init__.parsed_docstring.fields)==3 + ''.join(gettext(__init__.parsed_docstring.to_node()))) # type:ignore + assert len(__init__.parsed_docstring.fields)==3 # type:ignore assert re.match( r'''docstring of number A\sattr.ib\(factory=list\)List of ints\sattr.ib\(converter=convert_paths, factory=list\)Uses a converter''', - ' '.join(text for text in (''.join(gettext(f.body().to_node())) for f in __init__.parsed_docstring.fields)) + ' '.join(text for text in (''.join(gettext(f.body().to_node())) for f in __init__.parsed_docstring.fields)) # type:ignore ) @attrs_systemcls_param diff --git a/pydoctor/test/test_sphinx.py b/pydoctor/test/test_sphinx.py index d05274c04..d71e0caf5 100644 --- a/pydoctor/test/test_sphinx.py +++ b/pydoctor/test/test_sphinx.py @@ -110,7 +110,7 @@ def test_generate_empty_functional() -> None: @contextmanager def openFileForWriting(path: str) -> Iterator[io.BytesIO]: yield output - inv_writer._openFileForWriting = openFileForWriting # type: ignore[assignment] + inv_writer._openFileForWriting = openFileForWriting # type: ignore inv_writer.generate(subjects=[], basepath='base-path') From 607a89a2bc137ec40106b1b8f267c4aac9a10b51 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 14 Jul 2023 02:20:40 -0400 Subject: [PATCH 40/48] add changelog entry --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index acf43a21a..7e6b224c0 100644 --- a/README.rst +++ b/README.rst @@ -88,6 +88,12 @@ in development * Improve the class hierarchy such that it links top level names with intersphinx when possible. * Add highlighting when clicking on "View In Hierarchy" link from class page. * Recognize variadic generics type variables (PEP 646). +* Better ``attrs`` support: generate precise ``__init__`` method from analyzed fields, supports + principal ``attrs`` idioms: + - ``attr.s(auto_attribs, kw_only, auto_detect, init)``/``attrs.define(...)`` + - ``attr.ib(init, default, factory, converter, type, kw_only)``/``attrs.field(...)`` + - ``attr.Factory(list)`` + It does not support the decorators based syntax for setting the validator/factory/default or converter. pydoctor 23.4.1 ^^^^^^^^^^^^^^^ From 3e512f429037d87e25efab2cb6bc2cf6ed7f834f Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 14 Jul 2023 02:34:36 -0400 Subject: [PATCH 41/48] make it pass the tests on older versions of python as well. --- pydoctor/test/test_attrs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 85c1c3177..01d6a6d29 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -23,7 +23,7 @@ def assert_constructor(cls:model.Documentable, sig:str, assert cls.dataclassLike == attrs.ModuleVisitor.DATACLASS_LIKE_KIND constructor = cls.contents['__init__'] assert isinstance(constructor, model.Function) - assert flatten_text(pages.format_signature(constructor)) == sig + assert flatten_text(pages.format_signature(constructor)).replace(' ','') == sig.replace(' ','') if shortsig: assert epydoc2stan.format_constructor_short_text(constructor, forclass=cls) == shortsig From 11ba98ca843a23d2e3ac528c21a5166116043e59 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 14 Jul 2023 02:40:14 -0400 Subject: [PATCH 42/48] normal indentation --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 7e6b224c0..69e8fef7e 100644 --- a/README.rst +++ b/README.rst @@ -90,9 +90,9 @@ in development * Recognize variadic generics type variables (PEP 646). * Better ``attrs`` support: generate precise ``__init__`` method from analyzed fields, supports principal ``attrs`` idioms: - - ``attr.s(auto_attribs, kw_only, auto_detect, init)``/``attrs.define(...)`` - - ``attr.ib(init, default, factory, converter, type, kw_only)``/``attrs.field(...)`` - - ``attr.Factory(list)`` + - ``attr.s(auto_attribs, kw_only, auto_detect, init)``/``attrs.define(...)`` + - ``attr.ib(init, default, factory, converter, type, kw_only)``/``attrs.field(...)`` + - ``attr.Factory(list)`` It does not support the decorators based syntax for setting the validator/factory/default or converter. pydoctor 23.4.1 From 83adf804cd9ceba7e00c9f93f73ed39d77ade6e9 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 14 Jul 2023 11:02:08 -0400 Subject: [PATCH 43/48] skip type comment test on python < 3.8 --- pydoctor/test/test_attrs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 01d6a6d29..94bfbef75 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -1,4 +1,5 @@ import re +import sys from typing import Optional, Type from pydoctor import epydoc2stan, model @@ -573,6 +574,7 @@ class MyClass: mod = fromText(src, systemcls=systemcls) assert_constructor(mod.contents['MyClass'], '(self, a: int)') +@pytest.mark.skipif(sys.version_info < (3, 8), reason="type comment requires python 3.8 or later") @attrs_systemcls_param def test_type_comment_wins_over_factory_annotation(systemcls: Type[model.System]) -> None: src = '''\ From ce1e05f5102824c83b0b263dc4d012da9e7f8a0f Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 14 Jul 2023 11:02:24 -0400 Subject: [PATCH 44/48] minor changes --- README.rst | 6 +++--- pydoctor/astutils.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 69e8fef7e..1d9c305bb 100644 --- a/README.rst +++ b/README.rst @@ -90,9 +90,9 @@ in development * Recognize variadic generics type variables (PEP 646). * Better ``attrs`` support: generate precise ``__init__`` method from analyzed fields, supports principal ``attrs`` idioms: - - ``attr.s(auto_attribs, kw_only, auto_detect, init)``/``attrs.define(...)`` - - ``attr.ib(init, default, factory, converter, type, kw_only)``/``attrs.field(...)`` - - ``attr.Factory(list)`` + - ``attr.s(auto_attribs, kw_only, auto_detect, init)``/``attrs.define(...)`` + - ``attr.ib(init, default, factory, converter, type, kw_only)``/``attrs.field(...)`` + - ``attr.Factory(list)`` It does not support the decorators based syntax for setting the validator/factory/default or converter. pydoctor 23.4.1 diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index 636c7a701..e9c4497b7 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -456,7 +456,7 @@ class _V(enum.Enum): NoValue = enum.auto() _T = TypeVar('_T', bound=object) def _get_literal_arg(args:BoundArguments, name:str, - typecheck:'type[_T]|tuple[type[_T],...]') -> Union['Literal[_V.NoValue]', _T]: + typecheck:Union[Type[_T], Tuple[Type[_T],...]]) -> Union['Literal[_V.NoValue]', _T]: """ Helper function for L{get_literal_arg}. @@ -485,7 +485,7 @@ def _get_literal_arg(args:BoundArguments, name:str, return value #type:ignore def get_literal_arg(args:BoundArguments, name:str, default:_T, - typecheck:'type[_T]|tuple[type[_T],...]', + typecheck: Union[Type[_T], Tuple[Type[_T],...]], lineno:int, module: 'model.Module') -> _T: """ Retreive the literal value of an argument from the L{BoundArguments}. From 34d7bb3fb8197cf77169476b5eb49a61cafc0dc0 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Mon, 17 Jul 2023 23:14:26 -0400 Subject: [PATCH 45/48] remove unused imports --- pydoctor/astutils.py | 4 ++-- pydoctor/extensions/attrs.py | 1 - pydoctor/templatewriter/__init__.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pydoctor/astutils.py b/pydoctor/astutils.py index e9c4497b7..75331e187 100644 --- a/pydoctor/astutils.py +++ b/pydoctor/astutils.py @@ -7,8 +7,8 @@ import platform import sys from numbers import Number -from typing import Any, Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union, Type, TypeVar, Generic -from inspect import BoundArguments, Signature, Parameter +from typing import Any, Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union, Type, TypeVar +from inspect import BoundArguments, Signature import ast from pydoctor import visitor diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index edbd5ea26..d3449db18 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -19,7 +19,6 @@ from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval from pydoctor.epydoc2stan import parse_docstring -from pydoctor.epydoc.docutils import new_document, set_node_attributes diff --git a/pydoctor/templatewriter/__init__.py b/pydoctor/templatewriter/__init__.py index e9ad6ddec..7a158a57d 100644 --- a/pydoctor/templatewriter/__init__.py +++ b/pydoctor/templatewriter/__init__.py @@ -1,5 +1,5 @@ """Render pydoctor data as HTML.""" -from typing import Any, Iterable, Iterator, Optional, Union, cast, TYPE_CHECKING +from typing import Any, Iterable, Iterator, Optional, Union, TYPE_CHECKING if TYPE_CHECKING: from typing_extensions import Protocol, runtime_checkable else: From d4deb776701c18b59ccaf14a7d08919cd2081c2b Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 3 Nov 2023 16:28:03 -0400 Subject: [PATCH 46/48] Fix merge error --- pydoctor/model.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pydoctor/model.py b/pydoctor/model.py index b091c699e..e8ee1781f 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -1450,8 +1450,6 @@ def defaultPostProcess(system:'System') -> None: for cls in system.objectsOfType(Class): # Initiate the MROs cls._init_mro() - # Lookup of constructors - cls._init_constructors() # Compute subclasses for b in cls.baseobjects: From 4926d472a61557f6b123472017cd25bb25d02074 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 2 Jan 2024 19:03:08 -0500 Subject: [PATCH 47/48] Add comments --- pydoctor/extensions/attrs.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 92c03c488..eae67847f 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -1,6 +1,7 @@ """ -Support for L{attrs}. +Support for L{attrs} and other similar idioms, including L{dataclasses}, +L{typing.NamedTuple} and L{pydantic} models. """ import ast @@ -84,6 +85,9 @@ def _callable_return_type(dname:List[str], ctx:model.Documentable) -> Optional[a Note that the expression might not be fully resolvable in the new context since it can come from other modules. + + This is not type inference, we're simply looking up the name and. If it's + a function, we use the return annotation as is. """ r = ctx.resolveName('.'.join(dname)) if isinstance(r, model.Class): @@ -91,10 +95,15 @@ def _callable_return_type(dname:List[str], ctx:model.Documentable) -> Optional[a elif isinstance(r, model.Function): rtype = r.annotations.get('return') if rtype: + # TODO: Here the returned ast might not be in the same module + # as the attrs class, so the names might not be resolvable. + # So the right to do would be check whether it's defined in the same module + # and if not: use the fully qualified name instead so the linker will link to the + # object successfuly. return rtype elif r is None and len(dname)==1 and dname[0] in builtin_types: return astutils.dottedname2node(dname) - # TODO: we might be able to use the shpinx inventory yo check if the + # TODO: we might be able to use the shpinx inventory to check if the # provided callable is a class, in which case the class could be linked. return None From a1fff9781684b91206bf73fbed21d113d6fc4cbe Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 24 Mar 2024 13:25:11 -0400 Subject: [PATCH 48/48] WIP- merging dataclass, attrs and other utilities into a single module. --- pydoctor/extensions/_dataclass_like.py | 77 ------ pydoctor/extensions/attrs.py | 309 +++++++++++++++++-------- pydoctor/test/test_attrs.py | 227 +++++++++++++++++- 3 files changed, 431 insertions(+), 182 deletions(-) delete mode 100644 pydoctor/extensions/_dataclass_like.py diff --git a/pydoctor/extensions/_dataclass_like.py b/pydoctor/extensions/_dataclass_like.py deleted file mode 100644 index cbd9bf29c..000000000 --- a/pydoctor/extensions/_dataclass_like.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Dataclass-like libraries are all alike: -- They transform class variables into instance variable un certain conditions. -- They automatically provides a constructor method without having to define __init__. -""" -import ast -from abc import abstractmethod, ABC -from typing import Optional, Union -from pydoctor import astutils -from pydoctor.model import Module, Attribute, Class, Documentable -from pydoctor.extensions import ModuleVisitorExt, ClassMixin - -class DataclasLikeClass(ClassMixin): - dataclassLike:Optional[object] = None - -class DataclassLikeVisitor(ModuleVisitorExt, ABC): - - DATACLASS_LIKE_KIND:object = NotImplemented - - def __init__(self) -> None: - super().__init__() - assert self.DATACLASS_LIKE_KIND is not NotImplemented, "constant DATACLASS_LIKE_KIND should have a value" - - @abstractmethod - def isDataclassLike(self, cls:ast.ClassDef, mod:Module) -> Optional[object]: - """ - If this classdef adopts dataclass-like behaviour, returns an non-zero int, otherwise returns None. - Returned value is directly stored in the C{dataclassLike} attribute of the visited class. - Used to determine whether L{transformClassVar} method should be called for each class variables - in this class. - - The int value should be a constant representing the kind of dataclass-like this class implements. - Class decorated with @dataclass and @attr.s will have different non-zero C{dataclassLike} attribute. - """ - - @abstractmethod - def transformClassVar(self, cls:Class, attr:Attribute, - annotation:Optional[ast.expr], - value:Optional[ast.expr]) -> None: - """ - Transform this class variable into a instance variable. - This method is left abstract because it's not as simple as setting:: - attr.kind = model.DocumentableKind.INSTANCE_VARIABLE - """ - - def visit_ClassDef(self, node: ast.ClassDef) -> None: - cls = self.visitor.builder._stack[-1].contents.get(node.name) - if not isinstance(cls, Class): - return - assert isinstance(cls, DataclasLikeClass) - dataclassLikeKind = self.isDataclassLike(node, cls.module) - if dataclassLikeKind: - if not cls.dataclassLike: - cls.dataclassLike = dataclassLikeKind - else: - cls.report(f'class is both {cls.dataclassLike} and {dataclassLikeKind}') - - def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: - current = self.visitor.builder.current - - for dottedname in astutils.iterassign(node): - if dottedname and len(dottedname)==1: - # We consider single name assignment only - if not isinstance(current, Class): - continue - assert isinstance(current, DataclasLikeClass) - if not current.dataclassLike == self.DATACLASS_LIKE_KIND: - continue - target, = dottedname - attr: Optional[Documentable] = current.contents.get(target) - if not isinstance(attr, Attribute) or \ - astutils.is_using_typing_classvar(attr.annotation, current): - continue - annotation = node.annotation if isinstance(node, ast.AnnAssign) else None - self.transformClassVar(current, attr, annotation, node.value) - - visit_AnnAssign = visit_Assign diff --git a/pydoctor/extensions/attrs.py b/pydoctor/extensions/attrs.py index 256ceed16..bd048ee04 100644 --- a/pydoctor/extensions/attrs.py +++ b/pydoctor/extensions/attrs.py @@ -1,61 +1,171 @@ """ Support for L{attrs} and other similar idioms, including L{dataclasses}, -L{typing.NamedTuple} and L{pydantic} models. +L{typing.NamedTuple} and L{pydantic} models. Later called "AL" for 'Attrs Like' classes. """ +# Implementation of these utilities have been regouped in a single module in +# order to minimize code duplication; as a side effect the code has a greater complexity. + from __future__ import annotations import ast +from abc import abstractmethod, ABC import enum import inspect import copy +import dataclasses -from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, Union +from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, Union, TYPE_CHECKING +if TYPE_CHECKING: + from typing import NotRequired + from typing_extensions import TypedDict +else: + TypedDict = dict import attr from pydoctor import astbuilder, model, astutils, extensions, epydoc2stan -from pydoctor.extensions._dataclass_like import DataclasLikeClass, DataclassLikeVisitor from pydoctor.epydoc.markup import ParsedDocstring, Field from pydoctor.epydoc.markup.plaintext import ParsedPlaintextDocstring from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval +from pydoctor.extensions import ModuleVisitorExt, ClassMixin from pydoctor.epydoc2stan import parse_docstring - - +# TODO: insted of actually using signature() we should built the Signature manually from +# Parameter objects. attrs_decorator_signature = inspect.signature(attr.s) """Signature of the L{attr.s} class decorator.""" - attrib_signature = inspect.signature(attr.ib) """Signature of the L{attr.ib} function for defining class attributes.""" +dataclass_decorator_signature = inspect.signature(dataclasses.dataclass) +dataclass_field_signature = inspect.signature(dataclasses.field) +# This list is rather incomplete :/ builtin_types = frozenset(('frozenset', 'int', 'bytes', 'complex', 'list', 'tuple', 'set', 'dict', 'range')) -class AttrsDeco(enum.Enum): - CLASSIC = 1 +# The common process + +class AlOptions(TypedDict): + """ + Dictionary that may contain the following keys: + + - auto_attribs: bool|None + + L{True} if this class uses the C{auto_attribs} feature of the L{attrs} + library to automatically convert annotated fields into attributes. + + - kw_only: bool + + C{True} is this class uses C{kw_only} feature of L{attrs } library. + + - init: bool|None + + False if L{attrs } is not generating an __init__ method for this class. + + - auto_detect:bool + """ + auto_attribs: NotRequired[bool|None] + kw_only: NotRequired[bool] + init: NotRequired[bool|None] + auto_detect: NotRequired[bool] + +class AttrsLikeClass( model.Class): + def setup(self) -> None: + super().setup() + self._al_class_type: AlClassType = AlClassType.NOT_ATTRS_LIKE_CLASS + self._al_options = AlOptions() + + # these two attributes helps us infer the signature of the __init__ function + self._al_constructor_parameters: List[inspect.Parameter] = [] + self._al_constructor_annotations: Dict[str, Optional[ast.expr]] = {} + + # @abstractmethod + # def get_al_type(self, cls:ast.ClassDef, mod:model.Module) -> AlClassType: + # """ + # If this classdef adopts dataclass-like behaviour, returns an non-zero int, otherwise returns None. + # Returned value is directly stored in the C{dataclassLike} attribute of the visited class. + # Used to determine whether L{handleField} method should be called for each class variables + # in this class. + + # The int value should be a constant representing the kind of dataclass-like this class implements. + # Class decorated with @dataclass and @attr.s will have different non-zero C{dataclassLike} attribute. + # """ + + # @abstractmethod + # def handle_field(self, cls:model.Class, attr:model.Attribute, + # annotation:Optional[ast.expr], + # value:Optional[ast.expr]) -> None: + # """ + # Transform this class variable into a instance variable. + # This method is left abstract because it's not as simple as setting:: + # attr.kind = model.DocumentableKind.INSTANCE_VARIABLE + # """ + +class AlClassType(enum.Enum): + + NOT_ATTRS_LIKE_CLASS = 0 + """ + This class is just a regular class. + """ + + ATTRS_CLASSIC = 1 """ - attr.s like + L{attr.s} like. """ - NEW = 2 + ATTRS_NEW = 2 """ - attrs.define like + L{attrs.define} like. """ -def attrs_deco_kind(deco: ast.AST, module: model.Module) -> Optional[AttrsDeco]: - if isinstance(deco, ast.Call): - deco = deco.func - if astutils.node2fullname(deco, module) in ( - 'attr.s', 'attr.attrs', 'attr.attributes'): - return AttrsDeco.CLASSIC - elif astutils.node2fullname(deco, module) in ( - 'attr.mutable', 'attr.frozen', 'attr.define', - 'attrs.mutable', 'attrs.frozen', 'attrs.define', - ): - return AttrsDeco.NEW - return None + DATACLASS = 3 + """ + L{dataclasses.dataclass} like. + """ + + NAMEDTUPLE = 3 + """ + L{typing.NamedTuple} like. + """ + + PYDANTIC_MODEL = 4 + """ + L{pydantic.BaseModel} like. + """ + +def get_attrs_like_type(cls: ast.ClassDef, module: model.Module) -> AlClassType: + + types = [] + for dottedname, _ in astutils.iter_decorators(cls, module): + if dottedname in ( + 'attr.s', 'attr.attrs', 'attr.attributes'): + types.append(AlClassType.ATTRS_CLASSIC) + elif dottedname in ( + 'attr.mutable', 'attr.frozen', 'attr.define', + 'attrs.mutable', 'attrs.frozen', 'attrs.define', + ): + types.append(AlClassType.ATTRS_NEW) + elif dottedname in ('dataclasses.dataclass',): + types.append(AlClassType.DATACLASS) + + for basenode in cls.bases: + base_fullname = astutils.node2fullname(basenode, module) + if base_fullname in ('pydantic.BaseModel',): + types.append(AlClassType.PYDANTIC_MODEL) + elif base_fullname in ('typing.NamedTuple', + 'typing_extensions.NamedTuple'): + types.append(AlClassType.NAMEDTUPLE) + + if len(types)==1: + return types[0] + elif len(types)==0: + return AlClassType.NOT_ATTRS_LIKE_CLASS + + # TODO: warns because this class is detected as being of several distinct attrs like types :/ + return types[0] + def is_attrib(expr: Optional[ast.expr], ctx: model.Documentable) -> bool: """Does this expression return an C{attr.ib}?""" @@ -87,7 +197,7 @@ def _callable_return_type(dname:List[str], ctx:model.Documentable) -> Optional[a resolvable in the new context since it can come from other modules. This is not type inference, we're simply looking up the name and. If it's - a function, we use the return annotation as is. + a function, we use the return annotation as is (potentially with unresolved type variables). """ r = ctx.resolveName('.'.join(dname)) if isinstance(r, model.Class): @@ -208,35 +318,67 @@ def _f(assign:Union[ast.Assign, ast.AnnAssign]) -> bool: args=[], keywords=[], lineno=0,) _nothing = object() -class ModuleVisitor(DataclassLikeVisitor): +class ModuleVisitor(ModuleVisitorExt): - DATACLASS_LIKE_KIND = 'attrs class' + # def visit_ClassDef(self, node: ast.ClassDef) -> None: + # if dataclassLikeKind: + # if not cls._al_class_type: + # cls._al_class_type = dataclassLikeKind + # else: + # cls.report(f'class is both {cls._al_class_type} and {dataclassLikeKind}') + + def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None: + current = self.visitor.builder.current + + for dottedname in astutils.iterassign(node): + if dottedname and len(dottedname)==1: + # We consider single name assignment only + if not isinstance(current, model.Class): + continue + assert isinstance(current, AttrsLikeClass) + if current._al_class_type == AlClassType.NOT_ATTRS_LIKE_CLASS: + continue + target, = dottedname + attr: Optional[model.Documentable] = current.contents.get(target) + if not isinstance(attr, model.Attribute) or \ + astutils.is_using_typing_classvar(attr.annotation, current): + continue + annotation = node.annotation if isinstance(node, ast.AnnAssign) else None + self.handle_field(current, attr, annotation, node.value) + + visit_AnnAssign = visit_Assign def visit_ClassDef(self, node:ast.ClassDef) -> None: """ Called when a class definition is visited. """ - super().visit_ClassDef(node) - cls = self.visitor.builder._stack[-1].contents.get(node.name) - if not isinstance(cls, AttrsClass) or cls.dataclassLike != self.DATACLASS_LIKE_KIND: - # not an attrs class + if not isinstance(cls, model.Class): return + assert isinstance(cls, AttrsLikeClass) + al_type, al_options = self.get_al_type_and_options(node, cls.module) + + cls._al_class_type = al_type + cls._al_options = al_options + + if al_type == AlClassType.NOT_ATTRS_LIKE_CLASS: + # not an attrs like class + return + mod = cls.module try: attrs_deco = next(decnode for decnode in node.decorator_list - if attrs_deco_kind(decnode, mod)) + if get_attrs_like_type(decnode, mod)) except StopIteration: return # init the self argument - cls.attrs_constructor_parameters.append( + cls._al_constructor_parameters.append( inspect.Parameter('self', inspect.Parameter.POSITIONAL_OR_KEYWORD) ) - cls.attrs_constructor_annotations['self'] = None + cls._al_constructor_annotations['self'] = None - kind = attrs_deco_kind(attrs_deco, mod) attrs_args = astutils.safe_bind_args(attrs_decorator_signature, attrs_deco, mod) # init attrs options based on arguments and whether the devs are using @@ -247,31 +389,31 @@ def visit_ClassDef(self, node:ast.ClassDef) -> None: 'kw_only': (False, bool), 'auto_detect': (False, bool), } - if kind == AttrsDeco.NEW: + if al_type == AlClassType.ATTRS_NEW: attrs_param_spec['auto_attribs'] = (None, (bool, type(None))) attrs_param_spec['auto_detect'] = (True, bool) if not attrs_args: attrs_args = astutils.bind_args(attrs_decorator_signature, _fallback_attrs_call) - cls.attrs_options.update({name: astutils.get_literal_arg(attrs_args, name, default, + cls._al_options.update({name: astutils.get_literal_arg(attrs_args, name, default, typecheck, attrs_deco.lineno, mod ) for name, (default, typecheck) in attrs_param_spec.items()}) - if kind is AttrsDeco.NEW and cls.attrs_options['auto_attribs'] is None: + if al_type is AlClassType.ATTRS_NEW and cls._al_options['auto_attribs'] is None: fields = collect_fields(node, cls) # auto detect auto_attrib value - cls.attrs_options['auto_attribs'] = len(fields)>0 and \ + cls._al_options['auto_attribs'] = len(fields)>0 and \ not any(isinstance(a, ast.Assign) for a in fields) - def transformClassVar(self, cls: model.Class, + def handle_field(self, cls: model.Class, attr: model.Attribute, annotation:Optional[ast.expr], value:Optional[ast.expr]) -> None: - assert isinstance(cls, AttrsClass) + assert isinstance(cls, AttrsLikeClass) is_attrs_attrib = is_attrib(value, cls) - is_attrs_auto_attrib = cls.attrs_options.get('auto_attribs') and \ + is_attrs_auto_attrib = cls._al_options.get('auto_attribs') and \ not is_attrs_attrib and annotation is not None if not (is_attrs_attrib or is_attrs_auto_attrib): @@ -294,11 +436,11 @@ def transformClassVar(self, cls: model.Class, ('kw_only', False, bool),)} # Handle the auto-creation of the __init__ method. - if cls.attrs_options.get('init', _nothing) in (True, None) and \ + if cls._al_options.get('init', _nothing) in (True, None) and \ is_attrs_auto_attrib or attrib_args_value.get('init'): kind:inspect._ParameterKind = inspect.Parameter.POSITIONAL_OR_KEYWORD - if cls.attrs_options.get('kw_only') or attrib_args_value.get('kw_only'): + if cls._al_options.get('kw_only') or attrib_args_value.get('kw_only'): kind = inspect.Parameter.KEYWORD_ONLY attrs_default:Optional[ast.expr] = ast.Constant(value=..., lineno=attr.linenumber) @@ -320,6 +462,7 @@ def transformClassVar(self, cls: model.Class, # attrs strips the leading underscores from the parameter names, # since there is not such thing as a private parameter. + # This is not true for dataclasses and others! init_param_name = attr.name.lstrip('_') if attrib_args: @@ -330,8 +473,8 @@ def transformClassVar(self, cls: model.Class, else: constructor_annotation = attr.annotation - cls.attrs_constructor_annotations[init_param_name] = constructor_annotation - cls.attrs_constructor_parameters.append( + cls._al_constructor_annotations[init_param_name] = constructor_annotation + cls._al_constructor_parameters.append( inspect.Parameter( init_param_name, kind=kind, default=astbuilder._ValueFormatter(attrs_default, cls) @@ -339,56 +482,35 @@ def transformClassVar(self, cls: model.Class, annotation=astbuilder._AnnotationValueFormatter(constructor_annotation, cls) if constructor_annotation else inspect.Parameter.empty)) - def isDataclassLike(self, cls:ast.ClassDef, mod:model.Module) -> Optional[object]: - if any(attrs_deco_kind(dec, mod) for dec in cls.decorator_list): - return self.DATACLASS_LIKE_KIND - return None + def get_al_type_and_options(self, cls:ast.ClassDef, mod:model.Module) -> Tuple[AlClassType, AlOptions]: -class AttrsOptions(Dict[str, object]): - """ - Dictionary that may contain the following keys: - - - auto_attribs: bool|None - - L{True} if this class uses the C{auto_attribs} feature of the L{attrs} - library to automatically convert annotated fields into attributes. - - - kw_only: bool - - C{True} is this class uses C{kw_only} feature of L{attrs } library. - - - init: bool|None - - False if L{attrs } is not generating an __init__ method for this class. - - - auto_detect:bool - """ - -class AttrsClass(DataclasLikeClass, model.Class): - def setup(self) -> None: - super().setup() + try: + attrs_deco = next(decnode for decnode in cls.decorator_list + if get_attrs_like_type(decnode, mod)) + except StopIteration: + return - self.attrs_options = AttrsOptions() - self.attrs_constructor_parameters: List[inspect.Parameter] = [] - self.attrs_constructor_annotations: Dict[str, Optional[ast.expr]] = {} + # if any(get_attrs_like_type(dec, mod) for dec in cls.decorator_list): + # return self.DATACLASS_LIKE_KIND + # return None -def collect_inherited_constructor_params(cls:AttrsClass) -> Tuple[List[inspect.Parameter], +def collect_inherited_constructor_params(cls:AttrsLikeClass) -> Tuple[List[inspect.Parameter], Dict[str, Optional[ast.expr]]]: # see https://github.com/python-attrs/attrs/pull/635/files base_attrs:List[inspect.Parameter] = [] base_annotations:Dict[str, Optional[ast.expr]] = {} - own_param_names = cls.attrs_constructor_annotations + own_param_names = cls._al_constructor_annotations # Traverse the MRO and collect attributes. for base_cls in reversed(cls.mro(include_external=False, include_self=False)): - assert isinstance(base_cls, AttrsClass) - for p in base_cls.attrs_constructor_parameters[1:]: + assert isinstance(base_cls, AttrsLikeClass) + for p in base_cls._al_constructor_parameters[1:]: if p.name in own_param_names: continue base_attrs.append(p) - base_annotations[p.name] = base_cls.attrs_constructor_annotations[p.name] + base_annotations[p.name] = base_cls._al_constructor_annotations[p.name] # For each name, only keep the freshest definition i.e. the furthest at the # back. base_annotations is fine because it gets overwritten with every new @@ -403,7 +525,7 @@ def collect_inherited_constructor_params(cls:AttrsClass) -> Tuple[List[inspect.P return filtered, base_annotations -def attrs_constructor_docstring(cls:AttrsClass, constructor_signature:inspect.Signature) -> ParsedDocstring: +def attrs_constructor_docstring(cls:AttrsLikeClass, constructor_signature:inspect.Signature) -> ParsedDocstring: """ Get a docstring for the attrs generated constructor method """ @@ -430,13 +552,13 @@ def attrs_constructor_docstring(cls:AttrsClass, constructor_signature:inspect.Si def postProcess(system:model.System) -> None: - for cls in list(system.objectsOfType(AttrsClass)): + for cls in list(system.objectsOfType(AttrsLikeClass)): # by default attr.s() overrides any defined __init__ mehtod, whereas dataclasses. - if cls.dataclassLike == ModuleVisitor.DATACLASS_LIKE_KIND: + if cls._al_class_type != AlClassType.NOT_ATTRS_LIKE_CLASS: - if cls.attrs_options.get('init') is False or \ - cls.attrs_options.get('init', _nothing) is None and \ - cls.attrs_options.get('auto_detect') is True and \ + if cls._al_options.get('init') is False or \ + cls._al_options.get('init', _nothing) is None and \ + cls._al_options.get('auto_detect') is True and \ cls.contents.get('__init__'): continue @@ -452,14 +574,15 @@ def postProcess(system:model.System) -> None: # collect arguments from super classes attributes definitions. inherited_params, inherited_annotations = collect_inherited_constructor_params(cls) # don't forget to set the KEYWORD_ONLY flag on inherited parameters - if cls.attrs_options.get('kw_only') is True: + if cls._al_options.get('kw_only') is True: for p in inherited_params: - p._kind = inspect.Parameter.KEYWORD_ONLY #type:ignore[attr-defined] + p._kind = inspect.Parameter.KEYWORD_ONLY # type:ignore[attr-defined] # make sure that self is kept first. - parameters = [cls.attrs_constructor_parameters[0], - *inherited_params, *cls.attrs_constructor_parameters[1:]] - annotations:Dict[str, Optional[ast.expr]] = {'self': None, **inherited_annotations, - **cls.attrs_constructor_annotations} + parameters = [cls._al_constructor_parameters[0], + *inherited_params, *cls._al_constructor_parameters[1:]] + annotations: Dict[str, Optional[ast.expr]] = {'self': None, + **inherited_annotations, + **cls._al_constructor_annotations} # Re-ordering kw_only arguments at the end of the list for param in tuple(parameters): @@ -482,5 +605,5 @@ def postProcess(system:model.System) -> None: def setup_pydoctor_extension(r:extensions.ExtRegistrar) -> None: r.register_astbuilder_visitor(ModuleVisitor) - r.register_mixin(AttrsClass) + r.register_mixin(AttrsLikeClass) r.register_post_processor(postProcess) diff --git a/pydoctor/test/test_attrs.py b/pydoctor/test/test_attrs.py index 94bfbef75..fb1c6faf0 100644 --- a/pydoctor/test/test_attrs.py +++ b/pydoctor/test/test_attrs.py @@ -20,8 +20,8 @@ def assert_constructor(cls:model.Documentable, sig:str, shortsig:Optional[str]=None) -> None: - assert isinstance(cls, attrs.AttrsClass) - assert cls.dataclassLike == attrs.ModuleVisitor.DATACLASS_LIKE_KIND + assert isinstance(cls, attrs.AttrsLikeClass) + assert cls._al_class_type == attrs.ModuleVisitor.DATACLASS_LIKE_KIND constructor = cls.contents['__init__'] assert isinstance(constructor, model.Function) assert flatten_text(pages.format_signature(constructor)).replace(' ','') == sig.replace(' ','') @@ -105,8 +105,8 @@ class C: d = 123 # ignored by auto_attribs because no annotation ''', modname='test', systemcls=systemcls) C = mod.contents['C'] - assert isinstance(C, attrs.AttrsClass) - assert C.attrs_options['auto_attribs'] == True + assert isinstance(C, attrs.AttrsLikeClass) + assert C._al_options['auto_attribs'] == True assert C.contents['a'].kind is model.DocumentableKind.INSTANCE_VARIABLE assert C.contents['b'].kind is model.DocumentableKind.INSTANCE_VARIABLE assert C.contents['c'].kind is model.DocumentableKind.CLASS_VARIABLE @@ -160,11 +160,11 @@ class D(C): mod = fromText(src, systemcls=systemcls) assert capsys.readouterr().out == '' C = mod.contents['C'] - assert isinstance(C, attrs.AttrsClass) - assert C.attrs_options['init'] is None + assert isinstance(C, attrs.AttrsLikeClass) + assert C._al_options['init'] is None D = mod.contents['D'] - assert isinstance(D, attrs.AttrsClass) - assert D.attrs_options['init'] is False + assert isinstance(D, attrs.AttrsLikeClass) + assert D._al_options['init'] is False assert_constructor(C, '(self, c: int = 100, x: int = 1, b: int = 23)', 'C(c, x, b)') @@ -193,9 +193,9 @@ class C: ''' mod = fromText(src, systemcls=systemcls) C = mod.contents['C'] - assert isinstance(C, attrs.AttrsClass) - assert C.attrs_options['kw_only'] is True - assert C.attrs_options['init'] is None + assert isinstance(C, attrs.AttrsLikeClass) + assert C._al_options['kw_only'] is True + assert C._al_options['init'] is None assert_constructor(C, '(self, *, a, b: str)') # Test case for default factory @@ -637,4 +637,207 @@ def test_define_type_comment_not_auto_attribs(systemcls: Type[model.System]) -> class A: a = 0 #type:int''' mod = fromText(src, systemcls=systemcls) - assert_constructor(mod.contents['A'], '(self)') \ No newline at end of file + assert_constructor(mod.contents['A'], '(self)') + +@attrs_systemcls_param +def test_dataclass_basic(systemcls: Type[model.System]) -> None: + src = '''\ + import dataclasses + + @dataclasses.dataclass + class C: + x: int + y: int + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['C'], '(self, x: int, y: int)') + +@attrs_systemcls_param +def test_dataclass_defaults(systemcls: Type[model.System]) -> None: + src = '''\ + from dataclasses import dataclass + + @dataclass + class C: + name: str + age: int = 30 + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['C'], '(self, name: str, age: int = 30)') + +@attrs_systemcls_param +def test_dataclass_kw_only(systemcls: Type[model.System]) -> None: + src = '''\ + import dataclasses + + @dataclasses.dataclass(kw_only=True) + class C: + title: str + author: str + year: int + price: float + total_pages: int = 0 + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['C'], '(self, *, title: str, author: str, year: int, price: float, total_pages: int = 0)') + +@attrs_systemcls_param +def test_dataclass_kw_only_flag(systemcls: Type[model.System]) -> None: + src = '''\ + import dataclasses + + @dataclasses.dataclass + class C: + _: dataclasses.KY_ONLY + title: str + author: str + year: int + price: float + total_pages: int = 0 + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['C'], '(self, *, title: str, author: str, year: int, price: float, total_pages: int = 0)') + +@attrs_systemcls_param +def test_dataclass_factory(systemcls: Type[model.System]) -> None: + src = '''\ + import dataclasses + + @dataclasses.dataclass + class C: + width: int + height: int + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['C'], '(self, width: int, height: int)') + +@attrs_systemcls_param +def test_dataclass_no_constructor(systemcls: Type[model.System]) -> None: + src = '''\ + import dataclasses + @dataclasses.dataclass(init=False) + class C: + a: int + b: str + ''' + mod = fromText(src, systemcls=systemcls) + C = mod.contents['C'] + assert 'def __init__' not in C.source_code + +# Test case for dataclass with single inheritance: +@attrs_systemcls_param +def test_dataclass_constructor_single_inheritance(systemcls: Type[model.System]) -> None: + src = '''\ + import dataclasses + @dataclasses.dataclass + class Base: + a: int + @dataclasses.dataclass + class Derived(Base): + b: str + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['Derived'], '(self, a: int, b: str)', 'Derived(a, b)') + +# Test case for dataclass with multiple inheritance: +@attrs_systemcls_param +def test_dataclass_constructor_multiple_inheritance(systemcls: Type[model.System]) -> None: + src = '''\ + import dataclasses + @dataclasses.dataclass + class Base1: + a: int + @dataclasses.dataclass + class Base2: + b: str + @dataclasses.dataclass + class Derived(Base1, Base2): + c: float + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['Derived'], '(self, a: int, b: str, c: float)', 'Derived(a, b, c)') + +# Test case for dataclass with overridden attributes: +@attrs_systemcls_param +def test_dataclass_constructor_single_inheritance_overridden_attribute(systemcls: Type[model.System]) -> None: + src = '''\ + import dataclasses + @dataclasses.dataclass + class Base: + a: int + b: str = "default" + @dataclasses.dataclass + class Derived(Base): + b: str = "overridden" + c: float = 3.14 + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['Derived'], '(self, a: int, b: str = \'overridden\', c: float = 3.14)', 'Derived(a, b, c)') + +@attrs_systemcls_param +def test_dataclass_constructor_single_inheritance_traverse_subclasses(systemcls: Type[model.System]) -> None: + src = '''\ + import dataclasses + @dataclasses.dataclass + class FieldDesc: + name: Optional[str] = None + type: Optional[Tag] = None + body: Optional[Tag] = None + @dataclasses.dataclass + class _SignatureDesc(FieldDesc): + type_origin: Optional[object] = None + @dataclasses.dataclass + class ReturnDesc(_SignatureDesc):... + ''' + + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['ReturnDesc'], + '(self, name: Optional[str] = None, type: Optional[Tag] = None, body: Optional[Tag] = None, type_origin: Optional[object] = None)', + 'ReturnDesc(name, type, body, type_origin)') + +# Test case with dataclass field having init=False: +@attrs_systemcls_param +def test_dataclass_constructor_attribute_init_False(systemcls: Type[model.System]) -> None: + src = '''\ + import dataclasses + @dataclasses.dataclass + class MyClass: + a: int + b: str = dataclasses.field(init=False) + ''' + mod = fromText(src, systemcls=systemcls) + assert_constructor(mod.contents['MyClass'], '(self, a: int)') + +# Test case with dataclass field having kw_only=True: +@attrs_systemcls_param +def test_dataclass_constructor_attribute_kw_only_reorder(systemcls: Type[model.System], capsys:CapSys) -> None: + src = '''\ + import dataclasses + @dataclasses.dataclass + class MyClass: + a: int + b: str = dataclasses.field(kw_only=True) + c: float + ''' + mod = fromText(src, systemcls=systemcls) + assert not capsys.readouterr().out + assert_constructor(mod.contents['MyClass'], '(self, a: int, c: float, *, b: str)') + +@attrs_systemcls_param +def test_dataclass_constructor_kw_only_reordering_with_inheritence(systemcls: Type[model.System], capsys:CapSys) -> None: + # see https://docs.python.org/3/library/dataclasses.html#re-ordering-of-keyword-only-parameters-in-init + src = '''\ + @dataclass + class Base: + x: Any = 15.0 + y: int = field(kw_only=True, default=0) + w: int = field(kw_only=True, default=1) + + @dataclass + class D(Base): + z: int = 10 + t: int = field(kw_only=True, default=0) + ''' + mod = fromText(src, systemcls=systemcls) + assert not capsys.readouterr().out + assert_constructor(mod.contents['MyClass'], '(self, x: Any = 15.0, z: int = 10, *, y: int = 0, w: int = 1, t: int = 0)') \ No newline at end of file