Merge pull request #106 from bmwcarit/fe-update-dependencies

Update dependencies and pin elasticsearch version to <8

setup.py

@@ -25,7 +25,8 @@
 requires = [
     "arrow",
     "click",
-    "elasticsearch-dsl>=7.0.0",
+    "elasticsearch<8.0.0",
+    "elasticsearch-dsl>=7.0.0,<8.0.0",
     "pyyaml",
     "pyjwt",
     "pyzmq",

test-requirements.txt

@@ -1,80 +1,75 @@
 -i https://pypi.python.org/simple
-attrs==22.1.0 ; python_version >= '3.5'
-certifi==2022.9.24 ; python_version >= '3.6'
-cfgv==3.3.1 ; python_full_version >= '3.6.1'
-charset-normalizer==2.1.1 ; python_full_version >= '3.6.0'
-coverage[toml]==6.5.0 ; python_version >= '3.7'
-distlib==0.3.6
-exceptiongroup==1.0.4 ; python_version < '3.11'
-filelock==3.8.0 ; python_version >= '3.7'
-flake8==5.0.4
-flake8-docstrings==1.6.0
-flake8-import-order==0.18.1
-freezegun==1.2.2
-identify==2.5.9 ; python_version >= '3.7'
-idna==3.4 ; python_version >= '3.5'
-iniconfig==1.1.1
-mccabe==0.7.0 ; python_version >= '3.6'
-nodeenv==1.7.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
-packaging==21.3 ; python_version >= '3.6'
-platformdirs==2.5.4 ; python_version >= '3.7'
-pluggy==1.0.0 ; python_version >= '3.6'
-pre-commit==2.20.0
-pycodestyle==2.9.1 ; python_version >= '3.6'
-pydocstyle==6.1.1 ; python_version >= '3.6'
-pyflakes==2.5.0 ; python_version >= '3.6'
-pyparsing==3.0.9 ; python_full_version >= '3.6.8'
-pytest==7.2.0
-pytest-cov==4.0.0
-python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pyyaml==6.0 ; python_version >= '3.6'
-requests==2.28.1 ; python_version >= '3.7' and python_version < '4'
-requests-mock==1.10.0
-setuptools==65.6.0 ; python_version >= '3.7'
-six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+certifi==2023.11.17; python_version >= '3.6'
+cfgv==3.4.0; python_version >= '3.8'
+charset-normalizer==3.3.2; python_full_version >= '3.7.0'
+coverage[toml]==7.4.0; python_version >= '3.8'
+distlib==0.3.8
+filelock==3.13.1; python_version >= '3.8'
+flake8==7.0.0; python_full_version >= '3.8.1'
+flake8-docstrings==1.7.0; python_version >= '3.7'
+flake8-import-order==0.18.2
+freezegun==1.4.0; python_version >= '3.7'
+identify==2.5.33; python_version >= '3.8'
+idna==3.6; python_version >= '3.5'
+iniconfig==2.0.0; python_version >= '3.7'
+mccabe==0.7.0; python_version >= '3.6'
+nodeenv==1.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
+packaging==23.2; python_version >= '3.7'
+platformdirs==4.1.0; python_version >= '3.8'
+pluggy==1.3.0; python_version >= '3.8'
+pre-commit==3.6.0; python_version >= '3.9'
+pycodestyle==2.11.1; python_version >= '3.8'
+pydocstyle==6.3.0; python_version >= '3.6'
+pyflakes==3.2.0; python_version >= '3.8'
+pytest==7.4.4; python_version >= '3.7'
+pytest-cov==4.1.0; python_version >= '3.7'
+python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pyyaml==6.0.1; python_version >= '3.6'
+requests==2.31.0; python_version >= '3.7'
+requests-mock==1.11.0
+setuptools==69.0.3; python_version >= '3.8'
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 snowballstemmer==2.2.0
-toml==0.10.2 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-tomli==2.0.1 ; python_version < '3.11'
-urllib3==1.26.12 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'
-virtualenv==20.16.7 ; python_version >= '3.6'
-alabaster==0.7.12
-arrow==1.2.3 ; python_version >= '3.6'
-babel==2.11.0 ; python_version >= '3.6'
-bleach==5.0.1 ; python_version >= '3.7'
-cachelib==0.9.0 ; python_version >= '3.7'
-cffi==1.15.1
-click==8.1.3 ; python_version >= '3.7'
+urllib3==1.26.18; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+virtualenv==20.25.0; python_version >= '3.7'
+alabaster==0.7.13; python_version >= '3.6'
+arrow==1.3.0; python_version >= '3.8'
+babel==2.14.0; python_version >= '3.7'
+blinker==1.7.0; python_version >= '3.8'
+cachelib==0.10.2; python_version >= '3.7'
+cffi==1.16.0; python_version >= '3.8'
+click==8.1.7; python_version >= '3.7'
 cmarkgfm==2022.10.27
-cryptography==38.0.3 ; python_version >= '3.6'
-docutils==0.19 ; python_version >= '3.7'
-elasticsearch==7.17.7 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'
-elasticsearch-dsl==7.4.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-flask==2.2.2 ; python_version >= '3.7'
-gitdb==4.0.9 ; python_version >= '3.6'
-github3.py==3.2.0 ; python_version >= '3.6'
-gitpython==3.1.29 ; python_version >= '3.7'
-imagesize==1.4.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-itsdangerous==2.1.2 ; python_version >= '3.7'
-jinja2==3.1.2 ; python_version >= '3.7'
-markupsafe==2.1.1 ; python_version >= '3.7'
-pbr==5.11.0 ; python_version >= '2.6'
+cryptography==41.0.7; python_version >= '3.7'
+docutils==0.20.1; python_version >= '3.7'
+elasticsearch==7.17.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'
+elasticsearch-dsl==7.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+flask==3.0.0; python_version >= '3.8'
+gitdb==4.0.11; python_version >= '3.7'
+github3.py==4.0.1; python_version >= '3.7'
+gitpython==3.1.40; python_version >= '3.7'
+imagesize==1.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+itsdangerous==2.1.2; python_version >= '3.7'
+jinja2==3.1.2; python_version >= '3.7'
+markupsafe==2.1.3; python_version >= '3.7'
+nh3==0.2.15
+pbr==6.0.0; python_version >= '2.6'
 pycparser==2.21
-pygments==2.13.0 ; python_version >= '3.6'
-pyjwt==2.6.0 ; python_version >= '3.7'
-pytz==2022.6
-pyzmq==24.0.1 ; python_version >= '3.6'
-readme-renderer[md]==37.3 ; python_version >= '3.7'
-smmap==5.0.0 ; python_version >= '3.6'
-sphinx==5.3.0 ; python_version >= '3.6'
-sphinxcontrib-applehelp==1.0.2 ; python_version >= '3.5'
-sphinxcontrib-devhelp==1.0.2 ; python_version >= '3.5'
-sphinxcontrib-htmlhelp==2.0.0 ; python_version >= '3.6'
-sphinxcontrib-jsmath==1.0.1 ; python_version >= '3.5'
-sphinxcontrib-qthelp==1.0.3 ; python_version >= '3.5'
-sphinxcontrib-serializinghtml==1.1.5 ; python_version >= '3.5'
-tabulate==0.9.0 ; python_version >= '3.7'
-uritemplate==4.1.1 ; python_version >= '3.6'
-webencodings==0.5.1
-werkzeug==2.2.2 ; python_version >= '3.7'
+pygments==2.17.2; python_version >= '3.7'
+pyjwt[crypto]==2.8.0; python_version >= '3.7'
+pyzmq==25.1.2; python_version >= '3.6'
+readme-renderer[md]==42.0; python_version >= '3.8'
+smmap==5.0.1; python_version >= '3.7'
+sphinx==7.2.6; python_version >= '3.9'
+sphinxcontrib-applehelp==1.0.7; python_version >= '3.9'
+sphinxcontrib-devhelp==1.0.5; python_version >= '3.9'
+sphinxcontrib-htmlhelp==2.0.4; python_version >= '3.9'
+sphinxcontrib-jsmath==1.0.1; python_version >= '3.5'
+sphinxcontrib-qthelp==1.0.6; python_version >= '3.9'
+sphinxcontrib-serializinghtml==1.1.9; python_version >= '3.9'
+tabulate==0.9.0; python_version >= '3.7'
+types-python-dateutil==2.8.19.14
+uritemplate==4.1.1; python_version >= '3.6'
+werkzeug==3.0.1; python_version >= '3.8'
 -e .
-zuul-sphinx==0.6.0
+zuul-sphinx==0.7.0

tests/scraper/connections/test_github.py

@@ -38,7 +38,6 @@ def test_get_app_auth_headers():
 
 
 def test_get_installation_key(mock_github_api_endpoints):
-
     mock_github_api_endpoints(GITHUB_URL)
     # Initialize GitHubConnection
     gh_con = GitHubConnection(**GITHUB_CON_CONFIG)

tests/scraper/connections/test_init.py

@@ -88,7 +88,6 @@ def test_init_gerrit_con_invalid_webtype(patch_es):
 
 
 def test_init_github_con(patch_es, mock_github_api_endpoints):
-
     mock_github_api_endpoints("https://localhost/github")
     config = {
         "ES_HOST": "localhost",

tests/scraper/test_events.py

@@ -80,7 +80,6 @@ def test_event_installation_created(
 def test_event_installation_deleted(
     scrape_mock, patched_connections, payload_webhook_installation_deleted
 ):
-
     # Ensure that some repositories can be looked up for this installation as they
     # are not part of the payload for a delete event.
     with mock.patch(

tests/scraper/test_integration.py

@@ -68,7 +68,6 @@
 
 
 class MockContents:
-
     FILE = "file"
     DIR = "dir"
 
@@ -78,7 +77,6 @@ def __init__(self, path, type):
 
 
 class MockGitHubRepository(GitHubRepository):
-
     # TODO (felix): Improve the mocked repository class and allow mocking
     # single repositories for certain test cases. Currently, all test repos
     # are defined in this class definition.

tests/scraper/test_repo_parser.py

@@ -174,7 +174,7 @@ def test_parse(repo_data):
             '<span class="pre">mandatory_variable</span>'
             "</span>"
             '<a class="headerlink" href="#rolevar-mandatory_variable" '
-            'title="Permalink to this definition">'
+            'title="Link to this definition">'
             "¶</a><br /></dt>\n"
             "<dd><p>This variable is mandatory.</p>\n"
             "</dd></dl>\n"
@@ -185,7 +185,7 @@ def test_parse(repo_data):
             '<span class="pre">optional_variable</span>'
             "</span>"
             '<a class="headerlink" href="#rolevar-optional_variable" '
-            'title="Permalink to this definition">'
+            'title="Link to this definition">'
             '¶</a><br /><span class="pre">Default:</span> '
             '<code class="docutils literal notranslate"><span class="pre">'
             "some_value</span></code><br /></dt>\n"
@@ -198,7 +198,7 @@ def test_parse(repo_data):
             '<span class="pre">list_variable</span>'
             "</span>"
             '<a class="headerlink" href="#rolevar-list_variable" '
-            'title="Permalink to this definition">¶</a><br />'
+            'title="Link to this definition">¶</a><br />'
             '<span class="pre">Default:</span> '
             '<code class="docutils literal notranslate"><span class="pre">[]</span>'
             '</code><br /><span class="pre">Type:</span> <em><span class="pre">'

tests/test_doc.py

@@ -59,16 +59,16 @@ def test_render_sphinx_yaml_code(readme_yaml_code):
         '<div class="highlight-yaml notranslate"><div class="highlight">'
         '<pre><span></span><span class="p p-Indicator">-</span>'
         '<span class="w"> </span><span class="nt">job</span>'
-        '<span class="p">:</span><span class="w"></span>\n'
+        '<span class="p">:</span>\n'
         '<span class="w">    </span><span class="nt">name</span>'
         '<span class="p">:</span><span class="w"> </span>'
-        '<span class="l l-Scalar l-Scalar-Plain">foo</span><span class="w"></span>\n'
+        '<span class="l l-Scalar l-Scalar-Plain">foo</span>\n'
         '<span class="w">    </span><span class="nt">parent</span>'
         '<span class="p">:</span><span class="w"> </span>'
-        '<span class="l l-Scalar l-Scalar-Plain">bar</span><span class="w"></span>\n'
+        '<span class="l l-Scalar l-Scalar-Plain">bar</span>\n'
         '<span class="w">    </span><span class="nt">playbook</span>'
         '<span class="p">:</span><span class="w"> </span>'
-        '<span class="l l-Scalar l-Scalar-Plain">foo-bar.yaml</span><span class="w"></span>\n'
+        '<span class="l l-Scalar l-Scalar-Plain">foo-bar.yaml</span>\n'
         "</pre></div>\n"
         "</div>\n"
     )

tests/test_elasticsearch.py

@@ -196,7 +196,7 @@ def test_elasticsearch_write(elmock):
     zt.save()
 
     assert elmock.return_value.index.call_args == mock.call(
-        index="zuul-tenants", body=zt.to_dict()
+        index="zuul-tenants", document=zt.to_dict()
     )
 
 
@@ -207,5 +207,5 @@ def test_elasticsearch_write_with_prefix(elmock):
     zt.save()
 
     assert elmock.return_value.index.call_args == mock.call(
-        index="zubbi-zuul-tenants", body=zt.to_dict()
+        index="zubbi-zuul-tenants", document=zt.to_dict()
     )

tests/test_views.py

@@ -31,7 +31,6 @@ def test_views_reachable(flask_client, endpoint, expected):
 
 
 def test_detail_view(flask_client, es_client):
-
     # Build a simple ES response, containing a minimal role result
     response = {
         "hits": {

tox.ini

@@ -25,7 +25,7 @@ commands =
     python setup.py sdist bdist_wheel
 
 [testenv:update-requirements]
-whitelist_externals =
+allowlist_externals =
     bash
 deps =
     pipenv
@@ -35,8 +35,8 @@ commands =
     bash -c 'pipenv requirements --dev > "{toxinidir}/test-requirements.txt"'
 
 [testenv:pipenv]
-whitelist_externals =
-    {[testenv:update-requirements]whitelist_externals}
+allowlist_externals =
+    {[testenv:update-requirements]allowlist_externals}
 deps =
     {[testenv:update-requirements]deps}
 commands =

zubbi/doc.py

@@ -20,42 +20,57 @@
 import sys
 import tempfile
 
-from docutils.parsers.rst import Directive
 from readme_renderer import markdown
 from sphinx.application import Sphinx
 from sphinx.util import logging as sphinx_logging
 from sphinx.util.console import nocolor
-from sphinx.util.docutils import docutils_namespace, patch_docutils
+from sphinx.util.docutils import docutils_namespace, patch_docutils, SphinxDirective
 
 
 LOGGER = logging.getLogger(__name__)
 
 
-class ZubbiDirective(Directive):
+class ZubbiDirective(SphinxDirective):
     has_content = True
     required_arguments = 0
     optional_arguments = 1
     final_argument_whitespace = True
     logger = sphinx_logging.getLogger("ZubbiDirective")
 
-    def run(self):
 
+class SupportedOS(ZubbiDirective):
+    directive_name = "supported_os"
+
+    def run(self):
         if len(self.arguments) > 0:
             values = self.arguments[0].split(",")
-            for value in values:
-                # Log the value with directive_name for later extraction
-                self.logger.info("%s: %s", self.directive_name, value)
+            # Store the platforms in Sphinx' domain data, so we can extract
+            # them later on during the rendering process.
+            # NOTE (felix): The "correct" solution might be to define a own
+            # domain and let the domain create the initial domaindata, e.g.
+            # https://opendev.org/zuul/zuul-sphinx/src/branch/master/zuul_sphinx/zuul.py#L714
+            # However, as a simple solution, this should be sufficient.
+            zubbi_domain_data = self.env.domaindata.setdefault("zubbi", {})
+            zubbi_domain_data["platforms"] = [v.strip().lower() for v in values]
         # We don't want to render anything, so we return an empty list of nodes
         return []
 
 
-class SupportedOS(ZubbiDirective):
-    directive_name = "supported_os"
-
-
 class Reusable(ZubbiDirective):
     directive_name = "reusable"
 
+    def run(self):
+        reusable = False
+        if len(self.arguments) > 0:
+            reusable = self.arguments[0].strip().lower() in ["true", "yes"]
+        # Store the platforms in Sphinx' domain data, so we can extract
+        # them later on during the rendering process.
+        zubbi_domain_data = self.env.domaindata.setdefault("zubbi", {})
+        zubbi_domain_data["reusable"] = reusable
+
+        # We don't want to render anything, so we return an empty list of nodes
+        return []
+
 
 class SphinxBuildError(RuntimeError):
     pass
@@ -103,16 +118,11 @@ def render_sphinx(content):
             if app.statuscode:
                 raise SphinxBuildError
 
-        # Extract the platforms from the logger output
-        platforms = []
-        reusable = False
-        status_log.seek(0)
-        for line in status_log.readlines():
-            prefix, _, value = line.partition(":")
-            if prefix == SupportedOS.directive_name:
-                platforms.append(value.strip().lower())
-            elif prefix == Reusable.directive_name:
-                reusable = value.strip().lower() in ["true", "yes"]
+            # Extract the data from our custom directives from the domain data
+            zubbi_domain_data = app.env.domaindata.get("zubbi", {})
+            platforms = zubbi_domain_data.get("platforms", [])
+            reusable = zubbi_domain_data.get("reusable", False)
+
         with build_path.open() as build:
             html_parts = json.load(build)
 

zubbi/scraper/connections/gerrit.py

@@ -59,7 +59,6 @@ def build_directory_url(self, repo_name, directory_path):
 
 
 class GerritConnection(GitConnection):
-
     WEB_URL_BUILDERS = {"cgit": CGitUrlBuilder, "gitweb": GitwebUrlBuilder}
 
     # TODO (felix): Should we ensure that only the user that started zubbi has access rights

zubbi/scraper/connections/github.py

@@ -29,7 +29,6 @@
 
 class GitHubConnection:
     def __init__(self, url, app_id, app_key):
-
         self.base_url = url
         self.api_url = urljoin(url, "api/v3")
         self.graphql_url = urljoin(url, "api/graphql")