Merge branch 'master' into 623-better-constant-kind

README.rst

@@ -76,13 +76,21 @@ What's New?
 in development
 ^^^^^^^^^^^^^^
 
+* Do not show `**kwargs` when keywords are specifically documented with the `keyword` field
+  and no specific documentation is given for the `**kwargs` entry.
 * Fix annotation resolution edge cases: names are resolved in the context of the module 
   scope when possible, when impossible, the theoretical runtime scopes are used. A warning can
   be reported when an annotation name is ambiguous (can be resolved to different names 
   depending on the scope context) with option ``-v``.
 * Use stricter verifications before marking an attribute as constant. 
 * Do not trigger warnings when pydoctor cannot make sense of a potential constant attribute 
   (pydoctor is not a static checker).
+* Fix presentation of type aliases in string form.
+* Improve the AST colorizer to output less parenthesis when it's not required.
+* Fix colorization of dictionary unpacking.
+* Improve the class hierarchy such that it links top level names with intersphinx when possible.
+* Add highlighting when clicking on "View In Hierarchy" link from class page.
+* Recognize variadic generics type variables (PEP 646).
 
 pydoctor 23.4.1
 ^^^^^^^^^^^^^^^

docs/source/contrib.rst

@@ -123,6 +123,32 @@ Such new packages shouldn't get vendored. They need to be packaged in
 Debian. Best is to get in contact with the DPT to talk about about new
 requirements and the best way to get things done.
 
+Profiling pydoctor with austin and speedscope
+---------------------------------------------
+
+1. Install austin (https://github.com/P403n1x87/austin)
+2. Install austin-python (https://pypi.org/project/austin-python/)
+3. Run program under austin 
+
+   .. code::
+
+      $ sudo austin -i 1ms -C -o pydoctor.austin pydoctor <pydoctor args>
+
+4. Convert .austin to .speedscope (austin2speedscope comes from austin-python)
+
+   .. code::
+  
+    $ austin2speedscope pydoctor.austin pydoctor.speedscope
+
+
+5. Open https://speedscope.app and load pydoctor.speedscope into it.
+
+Note on sampling interval
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On our large repo I turn down the sampling interval from 100us to 1ms to make 
+the resulting ``.speedscope`` file a manageable size (15MB instead of 158MB which is too large to put into a gist.)
+
 Author Design Notes
 -------------------
 

docs/source/quickstart.rst

@@ -54,3 +54,15 @@ Output files are static HTML pages which require no extra server-side support.
 
 Here is a `GitHub Action example <publish-github-action.html>`_ to automatically
 publish your API documentation to your default GitHub Pages website.
+
+Return codes
+------------
+
+Pydoctor is a pretty verbose tool by default. It’s quite unlikely that you get a zero exit code on the first run. 
+But don’t worry, pydoctor should have produced useful HTML pages no matter your project design or docstrings. 
+
+Exit codes includes:
+- ``0``: All docstrings are well formatted (warnings may be printed).
+- ``1``: Pydoctor crashed with traceback (default Python behaviour).
+- ``2``: Some docstrings are mal formatted.
+- ``3``: Pydoctor detects some warnings and ``--warnings-as-errors`` is enabled.

pydoctor/astbuilder.py

@@ -95,7 +95,11 @@ class TypeAliasVisitorExt(extensions.ModuleVisitorExt):
     """
     def _isTypeVariable(self, ob: model.Attribute) -> bool:
         if ob.value is not None:
-            if isinstance(ob.value, ast.Call) and node2fullname(ob.value.func, ob) in ('typing.TypeVar', 'typing_extensions.TypeVar'):
+            if isinstance(ob.value, ast.Call) and \
+                node2fullname(ob.value.func, ob) in ('typing.TypeVar', 
+                                                     'typing_extensions.TypeVar',
+                                                     'typing.TypeVarTuple', 
+                                                     'typing_extensions.TypeVarTuple'):
                 return True
         return False
 
@@ -104,18 +108,11 @@ def _isTypeAlias(self, ob: model.Attribute) -> bool:
         Return C{True} if the Attribute is a type alias.
         """
         if ob.value is not None:
-
-            if is_using_annotations(ob.annotation, ('typing.TypeAlias', 'typing_extensions.TypeAlias'), ob):
-                try:
-                    ob.value = unstring_annotation(ob.value, ob)
-                except SyntaxError as e:
-                    ob.report(f"invalid type alias: {e}")
-                    return False
+            if is_using_annotations(ob.annotation, ('typing.TypeAlias', 
+                                                    'typing_extensions.TypeAlias'), ob):
                 return True
-
             if is_typing_annotation(ob.value, ob.parent):
                 return True
-
         return False
 
     def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None:
@@ -129,7 +126,12 @@ def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None:
                     return
                 if self._isTypeAlias(attr) is True:
                     attr.kind = model.DocumentableKind.TYPE_ALIAS
+                    # unstring type aliases
+                    attr.value = unstring_annotation(
+                        # this cast() is safe because _isTypeAlias() return True only if value is not None
+                        cast(ast.expr, attr.value), attr, section='type alias')
                 elif self._isTypeVariable(attr) is True:
+                    # TODO: unstring bound argument of type variables
                     attr.kind = model.DocumentableKind.TYPE_VARIABLE
 
     visit_AnnAssign = visit_Assign

pydoctor/astutils.py

@@ -195,7 +195,7 @@ def is_none_literal(node: ast.expr) -> bool:
     """Does this AST node represent the literal constant None?"""
     return isinstance(node, (ast.Constant, ast.NameConstant)) and node.value is None
 
-def unstring_annotation(node: ast.expr, ctx:'model.Documentable') -> ast.expr:
+def unstring_annotation(node: ast.expr, ctx:'model.Documentable', section:str='annotation') -> ast.expr:
     """Replace all strings in the given expression by parsed versions.
     @return: The unstringed node. If parsing fails, an error is logged
         and the original node is returned.
@@ -205,7 +205,7 @@ def unstring_annotation(node: ast.expr, ctx:'model.Documentable') -> ast.expr:
     except SyntaxError as ex:
         module = ctx.module
         assert module is not None
-        module.report(f'syntax error in annotation: {ex}', lineno_offset=node.lineno)
+        module.report(f'syntax error in {section}: {ex}', lineno_offset=node.lineno, section=section)
         return node
     else:
         assert isinstance(expr, ast.expr), expr

pydoctor/epydoc/markup/_pyval_repr.py

@@ -112,6 +112,13 @@ def restore(self, mark: _MarkedColorizerState) -> List[nodes.Node]:
         return trimmed
 
 # TODO: add support for comparators when needed. 
+# _OperatorDelimitier is needed for:
+# - IfExp
+# - UnaryOp
+# - BinOp, needs special handling for power operator
+# - Compare
+# - BoolOp
+# - Lambda
 class _OperatorDelimiter:
     """
     A context manager that can add enclosing delimiters to nested operators when needed. 
@@ -120,7 +127,7 @@ class _OperatorDelimiter:
     """
 
     def __init__(self, colorizer: 'PyvalColorizer', state: _ColorizerState, 
-                 node: Union[ast.UnaryOp, ast.BinOp, ast.BoolOp]) -> None:
+                 node: Union[ast.UnaryOp, ast.BinOp, ast.BoolOp],) -> None:
 
         self.discard = True
         """No parenthesis by default."""
@@ -133,12 +140,17 @@ def __init__(self, colorizer: 'PyvalColorizer', state: _ColorizerState,
         # See astutils.Parentage class, applied in PyvalColorizer._colorize_ast()
         parent_node: Optional[ast.AST] = getattr(node, 'parent', None)
 
-        if isinstance(parent_node, (ast.UnaryOp, ast.BinOp, ast.BoolOp)):
+        if parent_node:
             precedence = astor.op_util.get_op_precedence(node.op)
-            parent_precedence = astor.op_util.get_op_precedence(parent_node.op)
-            # Add parenthesis when precedences are equal to avoid confusions 
-            # and correctly handle the Pow special case without too much annoyance.
-            if precedence <= parent_precedence:
+            if isinstance(parent_node, (ast.UnaryOp, ast.BinOp, ast.BoolOp)):
+                parent_precedence = astor.op_util.get_op_precedence(parent_node.op)
+                if isinstance(parent_node.op, ast.Pow) or isinstance(parent_node, ast.BoolOp):
+                    parent_precedence+=1
+            else:
+                parent_precedence = colorizer.explicit_precedence.get(
+                    node, astor.op_util.Precedence.highest)
+
+            if precedence < parent_precedence:
                 self.discard = False
 
     def __enter__(self) -> '_OperatorDelimiter':
@@ -246,6 +258,9 @@ def __init__(self, linelen:Optional[int], maxlines:int, linebreakok:bool=True, r
         self.maxlines: Union[int, float] = maxlines if maxlines!=0 else float('inf')
         self.linebreakok = linebreakok
         self.refmap = refmap if refmap is not None else {}
+        # some edge cases require to compute the precedence ahead of time and can't be 
+        # easily done with access only to the parent node of some operators.
+        self.explicit_precedence:Dict[ast.AST, int] = {}
 
     #////////////////////////////////////////////////////////////
     # Colorization Tags & other constants
@@ -279,6 +294,10 @@ def __init__(self, linelen:Optional[int], maxlines:int, linebreakok:bool=True, r
 
     RE_COMPILE_SIGNATURE = signature(re.compile)
 
+    def _set_precedence(self, precedence:int, *node:ast.AST) -> None:
+        for n in node:
+            self.explicit_precedence[n] = precedence
+
     def colorize(self, pyval: Any) -> ColorizedPyvalRepr:
         """
         Entry Point.
@@ -336,10 +355,6 @@ def _colorize(self, pyval: Any, state: _ColorizerState) -> None:
         elif pyvaltype is frozenset:
             self._multiline(self._colorize_iter, pyval,
                             state, prefix='frozenset([', suffix='])')
-        elif pyvaltype is dict:
-            self._multiline(self._colorize_dict,
-                            list(pyval.items()),
-                            state, prefix='{', suffix='}')
         elif pyvaltype is list:
             self._multiline(self._colorize_iter, pyval, state, prefix='[', suffix=']')
         elif issubclass(pyvaltype, ast.AST):
@@ -432,15 +447,20 @@ def _colorize_iter(self, pyval: Iterable[Any], state: _ColorizerState,
         if suffix is not None:
             self._output(suffix, self.GROUP_TAG, state)
 
-    def _colorize_dict(self, items: Iterable[Tuple[Any, Any]], state: _ColorizerState, prefix: str, suffix: str) -> None:
+    def _colorize_ast_dict(self, items: Iterable[Tuple[Optional[ast.AST], ast.AST]], 
+                           state: _ColorizerState, prefix: str, suffix: str) -> None:
         self._output(prefix, self.GROUP_TAG, state)
         indent = state.charpos
         for i, (key, val) in enumerate(items):
             if i>=1:
                 self._insert_comma(indent, state)
             state.result.append(self.WORD_BREAK_OPPORTUNITY)
-            self._colorize(key, state)
-            self._output(': ', self.COLON_TAG, state)
+            if key:
+                self._set_precedence(astor.op_util.Precedence.Comma, val)
+                self._colorize(key, state)
+                self._output(': ', self.COLON_TAG, state)
+            else:
+                self._output('**', None, state)
             self._colorize(val, state)
         self._output(suffix, self.GROUP_TAG, state)
 
@@ -531,7 +551,7 @@ def _colorize_ast(self, pyval: ast.AST, state: _ColorizerState) -> None:
             self._multiline(self._colorize_iter, pyval.elts, state, prefix='set([', suffix='])')
         elif isinstance(pyval, ast.Dict):
             items = list(zip(pyval.keys, pyval.values))
-            self._multiline(self._colorize_dict, items, state, prefix='{', suffix='}')
+            self._multiline(self._colorize_ast_dict, items, state, prefix='{', suffix='}')
         elif isinstance(pyval, ast.Name):
             self._colorize_ast_name(pyval, state)
         elif isinstance(pyval, ast.Attribute):