Intersphinx features

.github/workflows/static.yaml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.8'
+        python-version: '3.10'
 
     - name: Install tox
       run: |

README.rst

@@ -83,6 +83,17 @@ This is the last major release to support Python 3.7.
 * `ExtRegistrar.register_post_processor()` now supports a `priority` argument that is an int.
   Highest priority callables will be called first during post-processing.
 * Fix too noisy ``--verbose`` mode (suppres some ambiguous annotations warnings).
+* Major improvements of the intersphinx integration:
+  - Pydoctor now supports linking to arbitrary intersphinx references with Sphinx role ``:external:``. 
+  - Other common Sphinx reference roles like ``:ref:``, ``:any:``, ``:class:``, ``py:*``, etc are now 
+    properly interpreted (instead of being simply stripping from the docstring).
+  - The ``--intersphinx`` option now supports the following format: ``[INVENTORY_NAME:]URL[:BASE_URL]``.
+    Where ``INVENTORY_NAME`` is a an arbitrary name used to filter ``:external:`` references, 
+    ``URL`` is an URL pointing to a ``objects.inv`` file (it can also be the base URL, ``/objects.inv`` will be added to the URL in this case).
+    It is recommended to always include the HTTP scheme in the intersphinx URLs. 
+  - The ``--intersphinx-file`` option has been added in order to load a local inventory file, this option
+    support the following format: ``[INVENTORY_NAME:]PATH:BASE_URL``. 
+    ``BASE_URL`` is the base for the generated links, it is mandatory if loading the inventory from a file.
 
 pydoctor 23.9.1
 ^^^^^^^^^^^^^^^

pydoctor/astbuilder.py

@@ -439,17 +439,10 @@ def _importNames(self, modname: str, names: Iterable[ast.alias]) -> None:
             _localNameToFullName[asname] = f'{modname}.{orgname}'
 
     def visit_Import(self, node: ast.Import) -> None:
-        """Process an import statement.
-
-        The grammar for the statement is roughly:
-
-        mod_as := DOTTEDNAME ['as' NAME]
-        import_stmt := 'import' mod_as (',' mod_as)*
+        """
+        Process an import statement.
 
-        and this is translated into a node which is an instance of Import wih
-        an attribute 'names', which is in turn a list of 2-tuples
-        (dotted_name, as_name) where as_name is None if there was no 'as foo'
-        part of the statement.
+        See L{import}. 
         """
         if not isinstance(self.builder.current, model.CanContainImportsDocumentable):
             # processing import statement in odd context

pydoctor/epydoc/markup/__init__.py

@@ -297,7 +297,11 @@ def link_to(self, target: str, label: "Flattenable") -> Tag:
         @return: The link, or just the label if the target was not found.
         """
 
-    def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag:
+    def link_xref(self, target: str, label: "Flattenable", lineno: int, *, 
+                  invname: Optional[str] = None,
+                  domain: Optional[str] = None,
+                  reftype: Optional[str] = None,
+                  external: bool = False) -> Tag:
         """
         Format a cross-reference link to a Python identifier.
         This will resolve the identifier to any reasonable target,
@@ -308,6 +312,14 @@ def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag:
         @param label: The label to show for the link.
         @param lineno: The line number within the docstring at which the
             crossreference is located.
+        @param invname: In the case of an intersphinx resolution, filters by
+            inventory name. 
+        @param domain: In the case of an intersphinx resolution, filters by
+            domain. 
+        @param reftype: In the case of an intersphinx resolution, filters by
+            reference type.
+        @param external: If True, forces the lookup to use intersphinx and 
+            ingnore local names.  
         @return: The link, or just the label if the target was not found.
             In either case, the returned top-level tag will be C{<code>}.
         """

pydoctor/epydoc/markup/epytext.py

@@ -1181,20 +1181,21 @@ def _colorize_link(link: Element, token: Token, end: int, errors: List[ParseErro
 
     # Clean up the target.  For URIs, assume http or mailto if they
     # don't specify (no relative urls)
-    target = re.sub(r'\s', '', target)
+    # we used to stip spaces from the target here but that's no good 
+    # since intersphinx targets can contain spaces.
     if link.tag=='uri':
+        target = re.sub(r'\s', '', target)
         if not re.match(r'\w+:', target):
             if re.match(r'\w+@(\w+)(\.\w+)*', target):
                 target = 'mailto:' + target
             else:
                 target = 'http://'+target
     elif link.tag=='link':
-        # Remove arg lists for functions (e.g., L{_colorize_link()})
-        target = re.sub(r'\(.*\)$', '', target)
-        if not re.match(r'^[a-zA-Z_]\w*(\.[a-zA-Z_]\w*)*$', target):
-            estr = "Bad link target."
-            errors.append(ColorizingError(estr, token, end))
-            return
+        # Here we used to process the target in order to remove arg lists for functions
+        # and validate it. But now this happens in node2stan.parse_reference().
+        # The target is not validated anymore since the intersphinx taget names can contain any kind of text.
+        # We simply normalize it.
+        target = re.sub(r'\s', ' ', target)
 
     # Construct the target element.
     target_elt = Element('target', target, lineno=str(token.startline))

pydoctor/epydoc/markup/restructuredtext.py

@@ -39,24 +39,32 @@
 the list.
 """
 from __future__ import annotations
+from contextlib import contextmanager
+from types import ModuleType
+
 __docformat__ = 'epytext en'
 
-from typing import Iterable, List, Optional, Sequence, Set, cast
-import re
-from docutils import nodes
+from typing import Any, Iterable, Iterator, List, Optional, Sequence, Set, Tuple, cast
 
+from docutils import nodes
+from docutils.utils import SystemMessage
 from docutils.core import publish_string
 from docutils.writers import Writer
-from docutils.parsers.rst.directives.admonitions import BaseAdmonition # type: ignore[import]
+from docutils.parsers.rst.directives.admonitions import BaseAdmonition # type: ignore[import-untyped]
 from docutils.readers.standalone import Reader as StandaloneReader
 from docutils.utils import Reporter
 from docutils.parsers.rst import Directive, directives
 from docutils.transforms import Transform, frontmatter
+from docutils.parsers.rst import roles
+import docutils.parsers.rst.states
 
 from pydoctor.epydoc.markup import Field, ParseError, ParsedDocstring, ParserFunction
 from pydoctor.epydoc.markup.plaintext import ParsedPlaintextDocstring
-from pydoctor.epydoc.docutils import new_document
+from pydoctor.epydoc.docutils import new_document, set_node_attributes
 from pydoctor.model import Documentable
+from pydoctor.sphinx import (ALL_SUPPORTED_ROLES, SUPPORTED_DEFAULT_REFTYPES, 
+                             SUPPORTED_DOMAINS, SUPPORTED_EXTERNAL_DOMAINS, 
+                             SUPPORTED_EXTERNAL_STD_REFTYPES, parse_domain_reftype)
 
 #: A dictionary whose keys are the "consolidated fields" that are
 #: recognized by epydoc; and whose values are the corresponding epydoc
@@ -93,18 +101,11 @@
     """
     writer = _DocumentPseudoWriter()
     reader = _EpydocReader(errors) # Outputs errors to the list.
-
-    # Credits: mhils - Maximilian Hils from the pdoc repository https://github.com/mitmproxy/pdoc
-    # Strip Sphinx interpreted text roles for code references: :obj:`foo` -> `foo`
-    docstring = re.sub(
-        r"(:py)?:(mod|func|data|const|class|meth|attr|exc|obj):", "", docstring
-    )
-
-    publish_string(docstring, writer=writer, reader=reader,
-                   settings_overrides={'report_level':10000,
-                                       'halt_level':10000,
-                                       'warning_stream':None})
-
+    with patch_docutils_role_function(errors):
+        publish_string(docstring, writer=writer, reader=reader,
+                       settings_overrides={'report_level':10000,
+                                           'halt_level':10000,
+                                           'warning_stream':None})
     document = writer.document
     visitor = _SplitFieldsTranslator(document, errors)
     document.walk(visitor)
@@ -498,6 +499,131 @@
                 'caption': directives.unchanged_required,
     }
 
+def parse_external(name: str) -> Tuple[Optional[str], Optional[str]]:
+    """
+    Returns a tuple: (inventory name, role)
+
+    @raises ValueError: If the format is invalid.
+    """
+    assert name.startswith('external'), name
+    # either we have an explicit inventory name, i.e,
+    # :external+inv:reftype:        or
+    # :external+inv:domain:reftype:
+    # or we look in all inventories, i.e.,
+    # :external:reftype:            or
+    # :external:domain:reftype:     or
+    # :external: 
+    suffix = name[9:]
+    if len(name) > len('external'):
+        if name[8] == '+':
+            parts = suffix.split(':', 1)
+            if len(parts) == 2:
+                inv_name, suffix = parts
+                if inv_name and suffix:
+                    return inv_name, suffix
+            elif len(parts) == 1:
+                inv_name, = parts
+                if inv_name:
+                    return inv_name, None
+        elif name[8] == ':' and suffix:
+            return None, suffix
+        msg = f'Malformed :external: role name: {name!r}'
+        raise ValueError(msg)
+    return None, None
+
+class LinkRole:
+    def __init__(self, errors: List[ParseError]) -> None:
+        self.errors = errors
+
+    # roles._RoleFn
+    def __call__(self, role: str, rawtext: str, text: str, lineno: int, 
+                inliner: docutils.parsers.rst.states.Inliner,
+                options:Any=None, content:Any=None) -> 'tuple[list[nodes.Node], list[nodes.Node]]':
+
+        # See https://www.sphinx-doc.org/en/master/usage/referencing.html
+        # and https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html
+        invname: Optional[str] = None
+        domain: Optional[str] = None
+        reftype: Optional[str] = None
+        external: bool = False
+        if role.startswith('external'):
+            try:
+                invname, suffix = parse_external(role)
+                if suffix is not None:
+                    domain, reftype = parse_domain_reftype(suffix)
+            except ValueError as e:
+                self.errors.append(ParseError(str(e), lineno, is_fatal=False))
+                return [], []
+            else:
+                external = True
+        elif role:
+            try:
+                domain, reftype = parse_domain_reftype(role)
+            except ValueError as e:
+                self.errors.append(ParseError(str(e), lineno, is_fatal=False))
+                return [], []
+
+        if reftype in SUPPORTED_DOMAINS and domain is None:
+            self.errors.append(ParseError('Malformed role name, domain is missing reference type', 
+                                          lineno, is_fatal=False))
+            return [], []
+
+        if reftype in SUPPORTED_DEFAULT_REFTYPES:
+            reftype = None
+
+        if reftype in SUPPORTED_EXTERNAL_STD_REFTYPES and domain is None:
+            external = True
+            domain = 'std'
+
+        if domain in SUPPORTED_EXTERNAL_DOMAINS:
+            external = True
+
+        text_node = nodes.Text(text)
+        node = nodes.title_reference(rawtext, '', 
+                                    invname=invname,
+                                    domain=domain,
+                                    reftype=reftype,
+                                    external=external,
+                                    lineno=lineno)
+
+        set_node_attributes(node, children=[text_node], document=inliner.document) # type: ignore
+        return [node], []
+
+@contextmanager
+def patch_docutils_role_function(errors:List[ParseError]) -> Iterator[None]:
+    r"""
+    Like sphinx, we are patching the L{docutils.parsers.rst.roles.role} function. 
+    This function is a factory for role handlers functions. In order to handle any kind
+    of roles names like C{:external+python:doc:`something`} (the role here is C{external+python:doc}, 
+    we need to patch this function because Docutils only handles extact matches...
+
+    Tip: To list roles contained in a given inventory, use the following command::
+
+        python3 -m sphinx.ext.intersphinx https://docs.python.org/3/objects.inv | grep -v '^\s'
+
+    """
+
+    old_role = roles.role
+
+    def new_role(role_name: str, language_module: ModuleType, 
+                 lineno: int, reporter: Reporter) -> 'tuple[nodes._RoleFn, list[SystemMessage]]':
+
+        if role_name in ALL_SUPPORTED_ROLES or any(
+            role_name.startswith(f'{n}:') for n in ALL_SUPPORTED_ROLES) or \
+            role_name.startswith('external+'): # 'external+' is a special case
+            return LinkRole(errors), []
+
+        return old_role(role_name, language_module, lineno, reporter) # type: ignore
+
+    roles.role = new_role
+    yield
+    roles.role = old_role
+
+# https://docutils.sourceforge.io/docs/ref/rst/directives.html#default-role
+# there is no possible code path that triggers messages from the default role, 
+# so that's ok to use an anonymous list here
+roles.register_local_role('default-role', LinkRole([]))
+
 directives.register_directive('python', PythonCodeDirective)
 directives.register_directive('code', DocutilsAndSphinxCodeBlockAdapter)
 directives.register_directive('code-block', DocutilsAndSphinxCodeBlockAdapter)