Drop python 3.6 and enable future annotations everywhere (#735)

* Remove python 3.6 from CI and add python 3.12

* Use python 3.12 rc3

* Remove all usages of ast.Str, ast.Bytes, etc in python 3.8 and later.

* Ignore some warnings for now since astor uses these deprecated classes.

* Add from __future__ import annotations everywhere

* Fix extract_docstring_linenum for python 3.7

* Use __instancecheck__ to prove a way to type check the docstring node stuff. Also minimizes the number of time we call get_str_value

* Use cpython 3.11, fixes #732

* Ensure that we can parse all files of the stdlib.

.github/workflows/unit.yaml

@@ -14,13 +14,13 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, 3.10.8, 3.11.0, pypy-3.6]
+        python-version: [pypy-3.7, 3.7, 3.8, 3.9, '3.10', 3.11, '3.12.0-rc.3']
         os: [ubuntu-20.04]
         include:
           - os: windows-latest
-            python-version: 3.6
+            python-version: 3.7
           - os: macos-latest
-            python-version: 3.6
+            python-version: 3.7
 
     steps:
     - uses: actions/checkout@v2
@@ -45,8 +45,8 @@ jobs:
       run: |
         tox -e test
 
-    - name: Run unit tests with latest Twisted version
-      if: matrix.python-version != '3.7' && matrix.python-version != '3.6' && matrix.python-version != 'pypy-3.6'
+    - name: Run unit tests with latest Twisted version (only for python 3.8 and later)
+      if: matrix.python-version != '3.7' && matrix.python-version != 'pypy-3.7'
       run: |
         tox -e test-latest-twisted
 

README.rst

@@ -76,6 +76,10 @@ What's New?
 in development
 ^^^^^^^^^^^^^^
 
+This is the last major release to support Python 3.7.
+
+* Drop support for Python 3.6
+* Add support for Python 3.12
 * `ExtRegistrar.register_post_processor()` now supports a `priority` argument that is an int.
   Highest priority callables will be called first during post-processing.
 

docs/tests/test_standard_library_docs.py

@@ -23,4 +23,11 @@ def test_std_lib_docs() -> None:
         elif entry.is_dir() and entry.joinpath('__init__.py').exists(): # Package
             assert BASE_DIR.joinpath('Lib.'+entry.name+'.html').exists()
 
-
+def test_std_lib_logs() -> None:
+    """
+    'Cannot parse file' do not appear too much.
+    This test expect a run.log file in cpython-output directory
+    """
+    log = (BASE_DIR / 'run.log').read_text()
+    assert log.count('cannot parse file') == 4
+

pydoctor/_configparser.py

@@ -20,6 +20,8 @@
 >>> parser = ArgumentParser(..., default_config_files=['./pyproject.toml', 'setup.cfg', 'my_super_tool.ini'], config_file_parser_class=MixedParser)
 
 """
+from __future__ import annotations
+
 import argparse
 from collections import OrderedDict
 import re

pydoctor/astbuilder.py

@@ -1,4 +1,5 @@
 """Convert ASTs into L{pydoctor.model.Documentable} instances."""
+from __future__ import annotations
 
 import ast
 import sys
@@ -16,8 +17,8 @@
 from pydoctor import epydoc2stan, model, node2stan, extensions, linker
 from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval
 from pydoctor.astutils import (is_none_literal, is_typing_annotation, is_using_annotations, is_using_typing_final, node2dottedname, node2fullname, 
-                               is__name__equals__main__, unstring_annotation, iterassign, extract_docstring_linenum, infer_type, get_parents, 
-                               NodeVisitor, Parentage)
+                               is__name__equals__main__, unstring_annotation, iterassign, extract_docstring_linenum, infer_type, get_parents,
+                               get_docstring_node, NodeVisitor, Parentage, Str)
 
 
 def parseFile(path: Path) -> ast.Module:
@@ -199,8 +200,9 @@ def visit_Module(self, node: ast.Module) -> None:
         Parentage().visit(node)
 
         self.builder.push(self.module, 0)
-        if len(node.body) > 0 and isinstance(node.body[0], ast.Expr) and isinstance(node.body[0].value, ast.Str):
-            self.module.setDocstring(node.body[0].value)
+        doc_node = get_docstring_node(node)
+        if doc_node is not None:
+            self.module.setDocstring(doc_node)
             epydoc2stan.extract_fields(self.module)
 
     def depart_Module(self, node: ast.Module) -> None:
@@ -257,8 +259,9 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None:
         cls._initialbaseobjects = initialbaseobjects
         cls._initialbases = initialbases
 
-        if len(node.body) > 0 and isinstance(node.body[0], ast.Expr) and isinstance(node.body[0].value, ast.Str):
-            cls.setDocstring(node.body[0].value)
+        doc_node = get_docstring_node(node)
+        if doc_node is not None:
+            cls.setDocstring(doc_node)
             epydoc2stan.extract_fields(cls)
 
         if node.decorator_list:
@@ -752,7 +755,7 @@ def visit_Assign(self, node: ast.Assign) -> None:
         if type_comment is None:
             annotation = None
         else:
-            annotation = unstring_annotation(ast.Str(type_comment, lineno=lineno), self.builder.current)
+            annotation = unstring_annotation(ast.Constant(type_comment, lineno=lineno), self.builder.current)
 
         for target in node.targets:
             if isinstance(target, ast.Tuple):
@@ -773,7 +776,7 @@ def visit_AugAssign(self, node:ast.AugAssign) -> None:
 
     def visit_Expr(self, node: ast.Expr) -> None:
         value = node.value
-        if isinstance(value, ast.Str):
+        if isinstance(value, Str):
             attr = self.builder.currentAttr
             if attr is not None:
                 attr.setDocstring(value)
@@ -803,11 +806,7 @@ def _handleFunctionDef(self,
             lineno = node.decorator_list[0].lineno
 
         # extracting docstring
-        docstring: Optional[ast.Str] = None
-        if len(node.body) > 0 and isinstance(node.body[0], ast.Expr) \
-                              and isinstance(node.body[0].value, ast.Str):
-            docstring = node.body[0].value
-
+        doc_node = get_docstring_node(node)
         func_name = node.name
 
         # determine the function's kind
@@ -840,7 +839,7 @@ def _handleFunctionDef(self,
 
         if is_property:
             # handle property and skip child nodes.
-            attr = self._handlePropertyDef(node, docstring, lineno)
+            attr = self._handlePropertyDef(node, doc_node, lineno)
             if is_classmethod:
                 attr.report(f'{attr.fullName()} is both property and classmethod')
             if is_staticmethod:
@@ -864,13 +863,13 @@ def _handleFunctionDef(self,
             func = self.builder.pushFunction(func_name, lineno)
 
         func.is_async = is_async
-        if docstring is not None:
+        if doc_node is not None:
             # Docstring not allowed on overload
             if is_overload_func:
-                docline = extract_docstring_linenum(docstring)
+                docline = extract_docstring_linenum(doc_node)
                 func.report(f'{func.fullName()} overload has docstring, unsupported', lineno_offset=docline-func.linenumber)
             else:
-                func.setDocstring(docstring)
+                func.setDocstring(doc_node)
         func.decorators = node.decorator_list
         if is_staticmethod:
             if is_classmethod:
@@ -942,7 +941,7 @@ def depart_FunctionDef(self, node: ast.FunctionDef) -> None:
 
     def _handlePropertyDef(self,
             node: Union[ast.AsyncFunctionDef, ast.FunctionDef],
-            docstring: Optional[ast.Str],
+            doc_node: Optional[Str],
             lineno: int
             ) -> model.Attribute:
 
@@ -951,8 +950,8 @@ def _handlePropertyDef(self,
                                          parent=self.builder.current)
         attr.setLineNumber(lineno)
 
-        if docstring is not None:
-            attr.setDocstring(docstring)
+        if doc_node is not None:
+            attr.setDocstring(doc_node)
             assert attr.docstring is not None
             pdoc = epydoc2stan.parse_docstring(attr, attr.docstring, attr)
             other_fields = []

pydoctor/astutils.py

@@ -1,12 +1,13 @@
 """
 Various bits of reusable code related to L{ast.AST} node processing.
 """
+from __future__ import annotations
 
 import inspect
 import platform
 import sys
 from numbers import Number
-from typing import Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union, cast
+from typing import Any, Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union, cast
 from inspect import BoundArguments, Signature
 import ast
 
@@ -137,6 +138,7 @@ def _is_str_constant(expr: ast.expr, s: str) -> bool:
         return isinstance(expr, ast.Constant) and expr.value == s
 else:
     # Before Python 3.8 "foo" was parsed as ast.Str.
+    # TODO: remove me when python3.7 is not supported anymore
     def get_str_value(expr:ast.expr) -> Optional[str]:
         if isinstance(expr, ast.Str):
             return expr.s
@@ -193,7 +195,11 @@ def is_using_annotations(expr: Optional[ast.AST],
 
 def is_none_literal(node: ast.expr) -> bool:
     """Does this AST node represent the literal constant None?"""
-    return isinstance(node, (ast.Constant, ast.NameConstant)) and node.value is None
+    if sys.version_info >= (3,8):
+        return isinstance(node, ast.Constant) and node.value is None
+    else:
+        # TODO: remove me when python3.7 is not supported anymore
+        return isinstance(node, (ast.Constant, ast.NameConstant)) and node.value is None
 
 def unstring_annotation(node: ast.expr, ctx:'model.Documentable', section:str='annotation') -> ast.expr:
     """Replace all strings in the given expression by parsed versions.
@@ -258,9 +264,10 @@ def visit_Constant(self, node: ast.Constant) -> ast.expr:
             return const
 
     # For Python < 3.8:
-
-    def visit_Str(self, node: ast.Str) -> ast.expr:
-        return ast.copy_location(self._parse_string(node.s), node)
+    if sys.version_info < (3,8):
+        # TODO: remove me when python3.7 is not supported anymore
+        def visit_Str(self, node: ast.Str) -> ast.expr:
+            return ast.copy_location(self._parse_string(node.s), node)
 
 TYPING_ALIAS = (
         "typing.Hashable",
@@ -357,14 +364,54 @@ def is_typing_annotation(node: ast.AST, ctx: 'model.Documentable') -> bool:
     return is_using_annotations(node, TYPING_ALIAS, ctx) or \
             is_using_annotations(node, SUBSCRIPTABLE_CLASSES_PEP585, ctx)
 
+def get_docstring_node(node: ast.AST) -> Str | None:
+    """
+    Return the docstring node for the given class, function or module
+    or None if no docstring can be found.
+    """
+    if not isinstance(node, (ast.AsyncFunctionDef, ast.FunctionDef, ast.ClassDef, ast.Module)) or not node.body:
+        return None
+    node = node.body[0]
+    if isinstance(node, ast.Expr):
+        if isinstance(node.value, Str):
+            return node.value
+    return None
 
 _string_lineno_is_end = sys.version_info < (3,8) \
                     and platform.python_implementation() != 'PyPy'
 """True iff the 'lineno' attribute of an AST string node points to the last
 line in the string, rather than the first line.
 """
 
-def extract_docstring_linenum(node: ast.Str) -> int:
+
+class _StrMeta(type):
+    if sys.version_info >= (3,8):
+        def __instancecheck__(self, instance: object) -> bool:
+            if isinstance(instance, ast.expr):
+                return get_str_value(instance) is not None
+            return False
+    else:
+        # TODO: remove me when python3.7 is not supported
+        def __instancecheck__(self, instance: object) -> bool:
+            return isinstance(instance, ast.Str)
+
+class Str(ast.expr, metaclass=_StrMeta):
+    """
+    Wraps ast.Constant/ast.Str for `isinstance` checks and annotations. 
+    Ensures that the value is actually a string.
+    Do not try to instanciate this class.
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        raise TypeError(f'{Str.__qualname__} cannot be instanciated')
+
+    if sys.version_info >= (3,8):
+        value: str
+    else:
+        # TODO: remove me when python3.7 is not supported
+        s: str
+
+def extract_docstring_linenum(node: Str) -> int:
     r"""
     In older CPython versions, the AST only tells us the end line
     number and we must approximate the start line number.
@@ -374,7 +421,11 @@ def extract_docstring_linenum(node: ast.Str) -> int:
     Leading blank lines are stripped by cleandoc(), so we must
     return the line number of the first non-blank line.
     """
-    doc = node.s
+    if sys.version_info >= (3,8):
+        doc = node.value
+    else:
+        # TODO: remove me when python3.7 is not supported
+        doc = node.s
     lineno = node.lineno
     if _string_lineno_is_end:
         # In older CPython versions, the AST only tells us the end line
@@ -393,16 +444,21 @@ def extract_docstring_linenum(node: ast.Str) -> int:
 
     return lineno
 
-def extract_docstring(node: ast.Str) -> Tuple[int, str]:
+def extract_docstring(node: Str) -> Tuple[int, str]:
     """
     Extract docstring information from an ast node that represents the docstring.
 
     @returns: 
         - The line number of the first non-blank line of the docsring. See L{extract_docstring_linenum}.
         - The docstring to be parsed, cleaned by L{inspect.cleandoc}.
     """
+    if sys.version_info >= (3,8):
+        value = node.value
+    else:
+        # TODO: remove me when python3.7 is not supported
+        value = node.s
     lineno = extract_docstring_linenum(node)
-    return lineno, inspect.cleandoc(node.s)
+    return lineno, inspect.cleandoc(value)
 
 
 def infer_type(expr: ast.expr) -> Optional[ast.expr]:
@@ -435,7 +491,7 @@ def _annotation_for_value(value: object) -> Optional[ast.expr]:
                 ann_elem = ast.Tuple(elts=[ann_elem, ann_value])
         if ann_elem is not None:
             if name == 'tuple':
-                ann_elem = ast.Tuple(elts=[ann_elem, ast.Ellipsis()])
+                ann_elem = ast.Tuple(elts=[ann_elem, ast.Constant(value=...)])
             return ast.Subscript(value=ast.Name(id=name),
                                  slice=ast.Index(value=ann_elem))
     return ast.Name(id=name)

pydoctor/driver.py

@@ -1,4 +1,5 @@
 """The entry point."""
+from __future__ import annotations
 
 from typing import  Sequence
 import datetime

pydoctor/epydoc/doctest.py

@@ -4,10 +4,10 @@
 #
 # Created [06/28/03 02:52 AM]
 #
-
 """
 Syntax highlighting for blocks of Python code.
 """
+from __future__ import annotations
 
 __docformat__ = 'epytext en'
 

pydoctor/epydoc/docutils.py

@@ -1,6 +1,8 @@
 """
 Collection of helper functions and classes related to the creation and processing of L{docutils} nodes.
 """
+from __future__ import annotations
+
 from typing import Iterable, Iterator, Optional
 
 import optparse

pydoctor/epydoc/markup/__init__.py

@@ -4,7 +4,6 @@
 # A python documentation Module
 # Edward Loper
 #
-
 """
 Markup language support for docstrings.  Each submodule defines a
 parser for a single markup language.  These parsers convert an
@@ -31,6 +30,7 @@
 classes record information about the cause, location, and severity of
 each error.
 """
+from __future__ import annotations
 __docformat__ = 'epytext en'
 
 from typing import Callable, ContextManager, List, Optional, Sequence, Iterator, TYPE_CHECKING

pydoctor/epydoc/markup/_napoleon.py

@@ -2,6 +2,8 @@
 This module contains a class to wrap shared behaviour between 
 L{pydoctor.epydoc.markup.numpy} and L{pydoctor.epydoc.markup.google}. 
 """
+from __future__ import annotations
+
 from typing import List, Optional, Type
 
 from pydoctor.epydoc.markup import ParsedDocstring, ParseError, processtypes