Merge branch 'master' into 822-819-search-bar-improvments-and-help

README.rst

@@ -74,6 +74,8 @@ in development
 ^^^^^^^^^^^^^^
 
 * Drop Python 3.7 and support Python 3.13.
+* Implement canonical HTML element (``<link rel="canonical" href="..."/>``) to help search engines reduce outdated content. 
+  Enable this feature by passing the base URL of the API documentation with option ``--html-base-url``.
 * Improve collection of objects:
    - Document objects declared in the ``else`` block of 'if' statements (previously they were ignored).
    - Document objects declared in ``finalbody`` and ``else`` block of 'try' statements (previously they were ignored).
@@ -85,6 +87,10 @@ in development
 * Replace the deprecated dependency appdirs with platformdirs.
 * Fix WinError caused by the failure of the symlink creation process.
   Pydoctor should now run on windows without the need to be administrator.
+* Adjust the sphinx extension to support Sphinx 8.1. The entries dynamically added to the intersphinx config
+  from the ``pydoctor_url_path`` config option now includes a project name which defaults to 'main' (instead of putting None), 
+  use mapping instead of a list to define your own project name.
+* Improve the themes so the adds injected by ReadTheDocs are rendered with the correct width and do not overlap too much with the main content.
 
 pydoctor 24.3.3
 ^^^^^^^^^^^^^^^

docs/source/conf.py

@@ -99,6 +99,7 @@
 pydoctor_args = {
     'main': [
         '--html-output={outdir}/api/',  # Make sure to have a trailing delimiter for better usage coverage.
+        '--html-base-url=https://pydoctor.readthedocs.io/en/latest/api',
         '--project-name=pydoctor',
         f'--project-version={version}',
         '--docformat=epytext', 
@@ -108,6 +109,7 @@
         ] + _common_args,
     'custom_template_demo': [
         '--html-output={outdir}/custom_template_demo/',
+        '--html-base-url=https://pydoctor.readthedocs.io/en/latest/custom_template_demo',
         f'--project-version={version}',
         f'--template-dir={_pydoctor_root}/docs/sample_template',
         f'{_pydoctor_root}/pydoctor',
@@ -116,6 +118,7 @@
               '-qqq' ], # we don't want to hear any warnings from this custom template demo.
     'epydoc_demo': [
         '--html-output={outdir}/docformat/epytext_demo',
+        '--html-base-url=https://pydoctor.readthedocs.io/en/latest/docformat/epytext_demo',
         '--project-name=pydoctor-epytext-demo',
         '--project-version=1.3.0',
         '--docformat=epytext',
@@ -126,6 +129,7 @@
         ] + _common_args,
     'restructuredtext_demo': [
         '--html-output={outdir}/docformat/restructuredtext_demo',
+        '--html-base-url=https://pydoctor.readthedocs.io/en/latest/docformat/restructuredtext_demo',
         '--project-name=pydoctor-restructuredtext-demo',
         '--project-version=1.0.0',
         '--docformat=restructuredtext',
@@ -136,6 +140,7 @@
         ] + _common_args,
     'numpy_demo': [ # no need to pass --docformat here, we use __docformat__
         '--html-output={outdir}/docformat/numpy_demo',
+         '--html-base-url=https://pydoctor.readthedocs.io/en/latest/docformat/numpy_demo',
         '--project-name=pydoctor-numpy-style-demo',
         '--project-version=1.0.0',
         '--project-url=../google-numpy.html',
@@ -145,6 +150,7 @@
         ] + _common_args,
     'google_demo': [
         '--html-output={outdir}/docformat/google_demo',
+         '--html-base-url=https://pydoctor.readthedocs.io/en/latest/docformat/google_demo',
         '--project-name=pydoctor-google-style-demo',
         '--project-version=1.0.0',
         '--docformat=google',

docs/source/publish-github-action.rst

@@ -5,7 +5,7 @@ Simple GitHub Action to publish API docs
 
 Here is an example of a simple GitHub Action to automatically
 generate your documentation with Pydoctor
-and publish it to your default GitHub Pages website.
+and publish it to your default GitHub Pages website when there is a push on the ``main`` branch.
 
 Just substitute `(projectname)` and `(packagedirectory)`
 with the appropriate information.
@@ -14,23 +14,24 @@ with the appropriate information.
 
     name: apidocs
     on:
-    - push
+      push:
+        branches: [main]
 
     jobs:
       deploy:
         runs-on: ubuntu-latest
 
         steps:
         - uses: actions/checkout@master
-        - name: Set up Python 3.8
+        - name: Set up Python 3.12
           uses: actions/setup-python@v2
           with:
-            python-version: 3.8
+            python-version: 3.12
 
         - name: Install requirements for documentation generation
           run: |
             python -m pip install --upgrade pip setuptools wheel
-            python -m pip install docutils pydoctor
+            python -m pip install pydoctor
 
         - name: Generate API documentation with pydoctor
           run: |
@@ -40,6 +41,7 @@ with the appropriate information.
                 --project-name=(projectname) \
                 --project-url=https://github.com/$GITHUB_REPOSITORY \
                 --html-viewsource-base=https://github.com/$GITHUB_REPOSITORY/tree/$GITHUB_SHA \
+                --html-base-url=https://$GITHUB_REPOSITORY_OWNER.github.io/${GITHUB_REPOSITORY#*/} \
                 --html-output=./apidocs \
                 --docformat=restructuredtext \
                 --intersphinx=https://docs.python.org/3/objects.inv \

docs/source/publish-readthedocs.rst

@@ -0,0 +1,51 @@
+:orphan:
+
+Simple ReadTheDocs config to publish API docs
+---------------------------------------------
+
+Here is an example of a simple ReadTheDocs integration to automatically
+generate your documentation with Pydoctor. 
+
+.. note:: This kind of integration should
+    not be confused with `Sphinx support <sphinx-integration.html>`_ that can also be used to run 
+    pydoctor inside ReadTheDocs as part of the standard Sphinx build process. 
+
+    This page, on the other hand, documents **how to simply run pydoctor 
+    and publish on ReadTheDocs** by using build customizations features.
+
+This example only includes a configuration file (``.readthedocs.yaml``), 
+but the repository must also have been 
+integrated to ReadTheDocs (by linking your Github account and importing your project for 
+instance or by `manual webhook configuration <https://stackoverflow.com/a/74959815>`_).
+
+The config file below assume you're cloning your repository with http(s) protocol 
+and that repository is a GitHub instance 
+(the value of ``--html-viewsource-base`` could vary depending on your git server). 
+
+Though, a similar process can be applied to Gitea, GitLab, Bitbucket ot others git servers.
+
+Just substitute `(projectname)` and `(packagedirectory)`
+with the appropriate information.
+
+.. code:: yaml
+
+    version: 2
+    build:
+    os: "ubuntu-22.04"
+    tools:
+        python: "3.10"
+    commands:
+        - pip install pydoctor
+        - | 
+        pydoctor \
+        --project-name=(projectname) \
+        --project-version=${READTHEDOCS_GIT_IDENTIFIER} \
+        --project-url=${READTHEDOCS_GIT_CLONE_URL%*.git} \
+        --html-viewsource-base=${READTHEDOCS_GIT_CLONE_URL%*.git}/tree/${READTHEDOCS_GIT_COMMIT_HASH} \
+        --html-base-url=${READTHEDOCS_CANONICAL_URL} \
+        --html-output $READTHEDOCS_OUTPUT/html/ \
+        --docformat=restructuredtext \
+        --intersphinx=https://docs.python.org/3/objects.inv \
+        ./(packagedirectory)
+
+`More on ReadTheDocs build customizations <https://docs.readthedocs.io/en/stable/build-customization.html>`_.

docs/source/quickstart.rst

@@ -26,9 +26,10 @@ The result looks like `this <api/index.html>`_.
 
     pydoctor \
         --project-name=pydoctor \
-        --project-version=1.2.0 \
+        --project-version=20.7.2 \
         --project-url=https://github.com/twisted/pydoctor/ \
         --html-viewsource-base=https://github.com/twisted/pydoctor/tree/20.7.2 \
+        --html-base-url=https://pydoctor.readthedocs.io/en/latest/api \
         --html-output=docs/api \
         --docformat=epytext \
         --intersphinx=https://docs.python.org/3/objects.inv \
@@ -53,6 +54,9 @@ Output files are static HTML pages which require no extra server-side support.
 Here is a `GitHub Action example <publish-github-action.html>`_ to automatically
 publish your API documentation to your default GitHub Pages website.
 
+Here is a `ReadTheDocs configuration <publish-readthedocs.html>`_ to automatically
+publish your API documentation to ReadTheDocs
+
 Return codes
 ------------
 

docs/tests/test.py

@@ -93,6 +93,7 @@ def test_page_contains_infos():
         - nav and links to modules, classes, names
         - js script source
         - pydoctor github link in the footer
+        - canonical link
     """
 
     infos = (f'<meta name="generator" content="pydoctor {__version__}"', 
@@ -101,7 +102,8 @@ def test_page_contains_infos():
               '<a href="classIndex.html"',
               '<a href="nameIndex.html"',
               '<script src="pydoctor.js" type="text/javascript"></script>',
-              '<a href="https://github.com/twisted/pydoctor/">pydoctor</a>',)
+              '<a href="https://github.com/twisted/pydoctor/">pydoctor</a>',
+              '<link rel="canonical" href="https://pydoctor.readthedocs.io/en/latest/api/pydoctor.driver.html"',)
 
     with open(BASE_DIR / 'api' / 'pydoctor.driver.html', 'r', encoding='utf-8') as stream:
         page = stream.read()
@@ -117,6 +119,7 @@ def test_custom_template_contains_infos():
         - pydoctor github link in the footer
         - the custom header
         - link to teh extra.css
+        - canonical link
     """
 
     infos = (f'<meta name="generator" content="pydoctor {__version__}"', 
@@ -126,7 +129,8 @@ def test_custom_template_contains_infos():
               '<a href="nameIndex.html"',
               '<a href="https://github.com/twisted/pydoctor/">pydoctor</a>',
               '<img src="https://twistedmatrix.com/trac/chrome/common/trac_banner.png" alt="Twisted" />',
-              '<link rel="stylesheet" type="text/css" href="extra.css" />',)
+              '<link rel="stylesheet" type="text/css" href="extra.css" />',
+              '<link rel="canonical" href="https://pydoctor.readthedocs.io/en/latest/custom_template_demo/index.html"',)
 
     with open(BASE_DIR / 'custom_template_demo' / 'index.html', 'r', encoding='utf-8') as stream:
         page = stream.read()

pydoctor/epydoc/markup/restructuredtext.py

@@ -41,7 +41,10 @@
 from __future__ import annotations
 __docformat__ = 'epytext en'
 
-from typing import Any, Iterable, List, Optional, Sequence, Set, cast
+from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Sequence, Set, cast
+if TYPE_CHECKING:
+    from typing import TypeAlias
+
 import re
 from docutils import nodes
 
@@ -190,7 +193,12 @@ def report(self, error: nodes.system_message) -> None:
 
         self._errors.append(ParseError(msg, linenum, is_fatal))
 
-class _DocumentPseudoWriter(Writer):
+if TYPE_CHECKING:
+    _StrWriter: TypeAlias = Writer[str]
+else:
+    _StrWriter = Writer
+
+class _DocumentPseudoWriter(_StrWriter):
     """
     A pseudo-writer for the docutils framework, that can be used to
     access the document itself.  The output of C{_DocumentPseudoWriter}

pydoctor/options.py

@@ -148,6 +148,11 @@ def get_parser() -> ArgumentParser:
             "The default behaviour auto detects most common providers like Github, Bitbucket, GitLab or SourceForge. "
             "But in some cases you might have to override the template string, for instance to make it work with git-web, use: "
             '--html-viewsource-template="{mod_source_href}#n{lineno}"'), metavar='SOURCETEMPLATE', default=Options.HTML_SOURCE_TEMPLATE_DEFAULT)
+    parser.add_argument(
+        '--html-base-url', dest='htmlbaseurl', 
+            help=("A base URL used to include a canonical link in every html page. "
+                  "This help search engine to link to the preferred version of "
+                  "a web page to prevent duplicated or oudated content. "), default=None, metavar='BASEURL', )
     parser.add_argument(
         '--buildtime', dest='buildtime',
         help=("Use the specified build time over the current time. "
@@ -297,6 +302,10 @@ def _convert_htmlwriter(s: str) -> Type['IWriter']:
         error(str(e))
 def _convert_privacy(l: List[str]) -> List[Tuple['model.PrivacyClass', str]]:
     return list(map(functools.partial(parse_privacy_tuple, opt='--privacy'), l))
+def _convert_htmlbaseurl(url:str | None) -> str | None:
+    if url and not url.endswith('/'): 
+        url += '/'
+    return url
 
 _RECOGNIZED_SOURCE_HREF = {
         # Sourceforge
@@ -361,6 +370,7 @@ class Options:
     htmlwriter:             Type['IWriter']                         = attr.ib(converter=_convert_htmlwriter)
     htmlsourcebase:         Optional[str]                           = attr.ib()
     htmlsourcetemplate:     str                                     = attr.ib()
+    htmlbaseurl:            str | None                              = attr.ib(converter=_convert_htmlbaseurl)
     buildtime:              Optional[str]                           = attr.ib()
     warnings_as_errors:     bool                                    = attr.ib()
     verbosity:              int                                     = attr.ib()

pydoctor/sphinx_ext/build_apidocs.py

@@ -113,7 +113,7 @@ def on_builder_inited(app: Sphinx) -> None:
             intersphinx_mapping = config.intersphinx_mapping
             url = url_path.format(**{'rtd_version': rtd_version})
             inv = (str(temp_path / 'objects.inv'),)
-            intersphinx_mapping[f'{key}-api-docs'] = (None, (url, inv))
+            intersphinx_mapping[f'{key}-api-docs'] = (key, (url, inv))
 
         # Build the API docs in temporary path.
         shutil.rmtree(temp_path, ignore_errors=True)

pydoctor/templatewriter/pages/__init__.py

@@ -7,6 +7,7 @@
 )
 import ast
 import abc
+from urllib.parse import urljoin
 
 from twisted.web.iweb import IRenderable, ITemplateLoader, IRequest
 from twisted.web.template import Element, Tag, renderer, tags
@@ -146,9 +147,19 @@ class Head(TemplateElement):
 
     filename = 'head.html'
 
-    def __init__(self, title: str, loader: ITemplateLoader) -> None:
+    def __init__(self, title: str, baseurl: str | None, pageurl: str, 
+                 loader: ITemplateLoader) -> None:
         super().__init__(loader)
         self._title = title
+        self._baseurl = baseurl
+        self._pageurl = pageurl
+
+    @renderer
+    def canonicalurl(self, request: IRequest, tag: Tag) -> Flattenable:
+        if not self._baseurl:
+            return ''
+        canonical_link = urljoin(self._baseurl, self._pageurl)
+        return tags.link(rel='canonical', href=canonical_link)
 
     @renderer
     def title(self, request: IRequest, tag: Tag) -> str:
@@ -171,6 +182,14 @@ def __init__(self, system: model.System,
         if not loader:
             loader = self.lookup_loader(template_lookup)
         super().__init__(loader)
+
+    @property
+    def page_url(self) -> str:
+        # This MUST be overriden in CommonPage
+        """
+        The relative page url
+        """
+        return self.filename
 
     def render(self, request: Optional[IRequest]) -> Tag:
         return tags.transparent(super().render(request)).fillSlots(**self.slot_map)
@@ -197,7 +216,8 @@ def title(self) -> str:
 
     @renderer
     def head(self, request: IRequest, tag: Tag) -> IRenderable:
-        return Head(self.title(), Head.lookup_loader(self.template_lookup))
+        return Head(self.title(), self.system.options.htmlbaseurl, self.page_url, 
+                    loader=Head.lookup_loader(self.template_lookup))
 
     @renderer
     def nav(self, request: IRequest, tag: Tag) -> IRenderable:

pydoctor/test/test_commandline.py

@@ -304,11 +304,30 @@ def test_index_hardlink(tmp_path: Path) -> None:
     assert not (tmp_path / 'basic.html').is_symlink()
     assert (tmp_path / 'basic.html').is_file()
 
+
 def test_apidocs_help(tmp_path: Path) -> None:
     """
     Checks that the help page is weel generated.
     """
     exit_code = driver.main(args=['--html-output', str(tmp_path), 'pydoctor/test/testpackages/basic/'])
     assert exit_code == 0
     help_page = (tmp_path / 'apidocs-help.html').read_text()
-    assert '<h2>Search</h2>' in help_page
+    assert '<h2>Search</h2>' in help_page
+
+def test_htmlbaseurl_option_all_pages(tmp_path: Path) -> None:
+    """
+    Check that the canonical link is included in all html pages, including summary pages.
+    """
+    exit_code = driver.main(args=[
+        '--html-base-url=https://example.com.abcde',
+        '--html-output', str(tmp_path), 'pydoctor/test/testpackages/basic/'])
+    assert exit_code == 0
+    for t in tmp_path.iterdir():
+        if not t.name.endswith('.html'):
+            continue
+        filename = t.name
+        if t.stem == 'basic':
+            filename = 'index.html' # since we have only one module it's linked as index.html
+        assert f'<link rel="canonical" href="https://example.com.abcde/{filename}"' in t.read_text(encoding='utf-8')
+
+