Tutorial directives (#195)

* add tutorial templates

* add installation directives

* remove tutorial dependencies from setup.py

* add tests for tutorial dependencies

* reformat

* remove debugging artefact

* skip tutorial test if dependencies are not met

* return tutorial dependencies to setup.py and add them to test_full

* skip tutorial tests if dependencies are not installed

* remove blank line

* run slow tests in test_coverage

* install devel_full in workflows

* update backup files

* remove unnecessary const

* copy tutorial code into a venv workspace

* remove markdown cells from installation replacement string

Also, add newline matching, set pip to quiet.

* replace !doclink with %doclink

* remove doc_prefix

* include slow tests in full test workflow

* remove `doc` as a dependency for tests

* remove redundant tutorial tests

* reformat

* add docker mark for tutorial tests

* exclude docker tests from non linux os

* revert removal of redundant tests (because of coverage)

.github/workflows/test_coverage.yml

@@ -41,7 +41,7 @@ jobs:
 
       - name: clean environment
         run: |
-          export backup_files=( tests tutorials .env_file makefile .coveragerc )
+          export backup_files=( tests tutorials .env_file makefile .coveragerc pytest.ini docs )
           mkdir /tmp/backup
           for i in "${backup_files[@]}" ; do mv "$i"  /tmp/backup ; done
           rm -rf ..?* .[!.]* *

.github/workflows/test_full.yml

@@ -48,7 +48,7 @@ jobs:
               source <(cat .env_file | sed 's/=/=/' | sed 's/^/export /')
               pytest --tb=long -vv --cache-clear --no-cov --allow-skip=telegram tests/
           else
-              pytest --tb=long -vv --cache-clear --no-cov --allow-skip=telegram,docker tests/
+              pytest -m "not docker" --tb=long -vv --cache-clear --no-cov --allow-skip=telegram,docker tests/
           fi
         shell: bash
   test_no_deps:

docs/source/conf.py

@@ -5,7 +5,7 @@
 # -- Path setup --------------------------------------------------------------
 
 sys.path.append(os.path.abspath("."))
-from utils.notebook import insert_installation_cell_into_py_tutorial  # noqa: E402
+from utils.notebook import py_percent_to_notebook  # noqa: E402
 from utils.generate_tutorials import generate_tutorial_links_for_notebook_creation  # noqa: E402
 from utils.regenerate_apiref import regenerate_apiref  # noqa: E402
 
@@ -87,7 +87,7 @@
 autosummary_generate_overwrite = False
 
 # Finding tutorials directories
-nbsphinx_custom_formats = {".py": insert_installation_cell_into_py_tutorial()}
+nbsphinx_custom_formats = {".py": py_percent_to_notebook}
 nbsphinx_prolog = """
 :tutorial_name: {{ env.docname }}
 """

docs/source/utils/notebook.py

@@ -1,42 +1,154 @@
-from jupytext import jupytext
+import re
+import abc
+from typing import ClassVar, Literal, Optional
 
+from pydantic import BaseModel
 
-def insert_installation_cell_into_py_tutorial():
+try:
+    from jupytext import jupytext
+except ImportError:
+    jupytext = None
+
+
+class ReplacePattern(BaseModel, abc.ABC):
     """
-    This function modifies a Jupyter notebook by inserting a code cell for installing 'dff' package
-    and its dependencies, and a markdown cell with instructions for the user. It uses the location of
-    the second cell in the notebook as a reference point to insert the new cells.
+    An interface for replace patterns.
     """
 
-    def inner(tutorial_text: str):
-        second_cell = tutorial_text.find("\n# %%", 5)
-        return jupytext.reads(
-            f"""{tutorial_text[:second_cell]}
+    @property
+    @abc.abstractmethod
+    def pattern(self) -> re.Pattern:
+        """
+        A regex pattern to replace in a text.
+        """
+        ...
+
+    @staticmethod
+    @abc.abstractmethod
+    def replacement_string(matchobj: re.Match) -> str:
+        """
+        Return a replacement string for a match object.
+
+        :param matchobj: A regex match object.
+        :return: A string to replace match with.
+        """
+        ...
+
+    @classmethod
+    def replace(cls, text: str) -> str:
+        """
+        Replace all instances of `pattern` in `text` with the result of `replacement_string`.
+
+        :param text: A text in which patterns are replaced.
+        :return: A string with patterns replaced.
+        """
+        return re.sub(cls.pattern, cls.replacement_string, text)
+
+
+class InstallationCell(ReplacePattern):
+    """
+    Replace installation cells directives.
+
+    Uncomment `# %pip install {}`, add a "quiet" flag, add a comment explaining the cell.
+    """
 
-# %% [markdown]
-\"\"\"
-__Installing dependencies__
-\"\"\"
+    pattern: ClassVar[re.Pattern] = re.compile("\n# %pip install (.*)\n")
 
+    @staticmethod
+    def replacement_string(matchobj: re.Match) -> str:
+        return f"""
 # %%
-!python3 -m pip install -q dff[tutorials]
-# Installs dff with dependencies for running tutorials
-# To install the minimal version of dff, use `pip install dff`
-# To install other options of dff, use `pip install dff[OPTION_NAME1,OPTION_NAME2]`
-# where OPTION_NAME can be one of the options from EXTRA_DEPENDENCIES.
-# e.g `pip install dff[ydb, mysql]` installs dff with dependencies for using Yandex Database and MySQL
-# EXTRA_DEPENDENCIES can be found in
-# https://github.com/deeppavlov/dialog_flow_framework/blob/dev/README.md#installation
-
-
-# %% [markdown]
-\"\"\"
-__Running tutorial__
-\"\"\"
-
-{tutorial_text[second_cell:]}
-""",
-            "py:percent",
-        )
-
-    return inner
+# installing dependencies
+%pip install -q {matchobj.group(1)}
+"""
+
+
+class DocumentationLink(ReplacePattern):
+    """
+    Replace documentation linking directives.
+
+    Replace strings of the `%doclink({args})` format with corresponding links to local files.
+
+    `args` is a comma-separated string of arguments to pass to the :py:meth:`.DocumentationLink.link_to_doc_page`.
+
+    So, `%doclink(arg1,arg2,arg3)` will be replaced with `link_to_doc_page(arg1, arg2, arg3)`, and
+    `%doclink(arg1,arg2)` will be replaced with `link_to_doc_page(arg1, arg2)`.
+
+    USAGE EXAMPLES
+    --------------
+
+    [link](%doclink(api,script.core.script))
+
+    [link](%doclink(api,script.core.script,Node))
+
+    [link](%doclink(tutorial,messengers.web_api_interface.4_streamlit_chat))
+
+    [link](%doclink(tutorial,messengers.web_api_interface.4_streamlit_chat,API-configuration))
+
+    [link](%doclink(guide,basic_conceptions))
+
+    [link](%doclink(guide,basic_conceptions,example-conversational-chat-bot))
+
+    """
+
+    pattern: ClassVar[re.Pattern] = re.compile(r"%doclink\((.+?)\)")
+
+    @staticmethod
+    def link_to_doc_page(
+        page_type: Literal["api", "tutorial", "guide"],
+        page: str,
+        anchor: Optional[str] = None,
+    ):
+        """
+        Create a link to a documentation page.
+
+        :param page_type:
+            Type of the documentation:
+
+                - "api" -- API reference
+                - "tutorial" -- Tutorials
+                - "guide" -- User guides
+
+        :param page:
+            Name of the page without the common prefix.
+
+            So, to link to keywords, pass "script.core.keywords" as page (omitting the "dff" prefix).
+
+            To link to the basic script tutorial, pass "script.core.1_basics" (without the "tutorials" prefix).
+
+            To link to the basic concepts guide, pass "basic_conceptions".
+        :param anchor:
+            An anchor on the page. (optional)
+
+            For the "api" type, use only the last part of the linked object.
+
+            So, to link to the `CLIMessengerInterface` class, pass "CLIMessengerInterface" only.
+
+            To link to a specific section of a tutorial or a guide, pass an anchor as-is (e.g. "introduction").
+        :return:
+            A link to the corresponding documentation part.
+        """
+        if page_type == "api":
+            return f"../apiref/dff.{page}.rst" + (f"#dff.{page}.{anchor}" if anchor is not None else "")
+        elif page_type == "tutorial":
+            return f"../tutorials/tutorials.{page}.py" + (f"#{anchor}" if anchor is not None else "")
+        elif page_type == "guide":
+            return f"../user_guides/{page}.rst" + (f"#{anchor}" if anchor is not None else "")
+
+    @staticmethod
+    def replacement_string(matchobj: re.Match) -> str:
+        args = matchobj.group(1).split(",")
+        return DocumentationLink.link_to_doc_page(*args)
+
+
+def apply_replace_patterns(text: str) -> str:
+    for cls in (InstallationCell, DocumentationLink):
+        text = cls.replace(text)
+
+    return text
+
+
+def py_percent_to_notebook(text: str):
+    if jupytext is None:
+        raise ModuleNotFoundError("`doc` dependencies are not installed.")
+    return jupytext.reads(apply_replace_patterns(text), "py:percent")

pytest.ini

@@ -2,5 +2,6 @@
 markers =
     docker: marks tests as requiring docker containers to work
     telegram: marks tests as requiring telegram client API token to work
+    slow: marks tests as slow (taking more than a minute to complete)
     all: reserved by allow-skip
     none: reserved by allow-skip

setup.py

@@ -100,26 +100,31 @@ def merge_req_lists(*req_lists: List[str]) -> List[str]:
         "pytest==7.4.0",
         "pytest-cov==4.1.0",
         "pytest-asyncio==0.21.0",
+        "pytest_virtualenv==1.7.0",
         "flake8==6.1.0",
         "click==8.1.3",
         "black==23.7.0",
         "isort==5.12.0",
-        "flask[async]==2.3.2",
-        "psutil==5.9.5",
-        "telethon==1.29.1",
-        "fastapi==0.101.0",
-        "uvicorn==0.23.1",
-        "websockets==11.0.2",
-        "locust==2.16.1",
-        "streamlit==1.25.0",
-        "streamlit-chat==0.1.1",
     ],
     requests_requirements,
 )
 
+tutorial_dependencies = [
+    "flask[async]==2.3.2",
+    "psutil==5.9.5",
+    "telethon==1.29.1",
+    "fastapi==0.101.0",
+    "uvicorn==0.23.1",
+    "websockets==11.0.2",
+    "locust==2.16.1",
+    "streamlit==1.25.0",
+    "streamlit-chat==0.1.1",
+]
+
 tests_full = merge_req_lists(
     full,
     test_requirements,
+    tutorial_dependencies,
 )
 
 doc = merge_req_lists(
@@ -171,7 +176,6 @@ def merge_req_lists(*req_lists: List[str]) -> List[str]:
     "full": full,  # full dependencies including all options above
     "tests": test_requirements,  # dependencies for running tests
     "test_full": tests_full,  # full dependencies for running all tests (all options above)
-    "tutorials": tests_full,  # dependencies for running tutorials (all options above)
     "devel": devel,  # dependencies for development
     "doc": doc,  # dependencies for documentation
     "devel_full": devel_full,  # full dependencies for development (all options above)

tests/pipeline/test_tutorials.py

@@ -26,7 +26,10 @@
     ],
 )
 def test_tutorials(tutorial_module_name: str):
-    tutorial_module = importlib.import_module(f"tutorials.{dot_path_to_addon}.{tutorial_module_name}")
+    try:
+        tutorial_module = importlib.import_module(f"tutorials.{dot_path_to_addon}.{tutorial_module_name}")
+    except ModuleNotFoundError as e:
+        pytest.skip(f"dependencies unavailable: {e.msg}")
     if tutorial_module_name == "6_custom_messenger_interface":
         happy_path = tuple(
             (req, Message(misc={"webpage": tutorial_module.construct_webpage_by_response(res.text)}))

tests/tutorials/test_tutorials.py

@@ -0,0 +1,49 @@
+from typing import TYPE_CHECKING
+import re
+from pathlib import Path
+
+import pytest
+
+if TYPE_CHECKING:
+    from pytest_virtualenv import VirtualEnv
+
+from docs.source.utils.notebook import InstallationCell
+
+
+PROJECT_ROOT_DIR = Path(__file__).parent.parent.parent
+DFF_TUTORIAL_PY_FILES = map(str, (PROJECT_ROOT_DIR / "tutorials").glob("./**/*.py"))
+
+
+def check_tutorial_dependencies(venv: "VirtualEnv", tutorial_source_code: str):
+    """
+    Install dependencies required by a tutorial in `venv` and run the tutorial.
+
+    :param venv: Virtual environment to run the tutorial in.
+    :param tutorial_source_code: Source code of the tutorial (unmodified by `apply_replace_patterns`).
+    :param tmp_path: Temporary path to save the tutorial to.
+    :return:
+    """
+    tutorial_path = venv.workspace / "tutorial.py"
+
+    venv.env["DISABLE_INTERACTIVE_MODE"] = "1"
+
+    with open(tutorial_path, "w") as fd:
+        fd.write(tutorial_source_code)
+
+    for deps in re.findall(InstallationCell.pattern, tutorial_source_code):
+        venv.run(f"python -m pip install {deps}", check_rc=True)
+
+    venv.run(f"python {tutorial_path}", check_rc=True)
+
+
+@pytest.mark.parametrize("dff_tutorial_py_file", DFF_TUTORIAL_PY_FILES)
+@pytest.mark.slow
+@pytest.mark.docker
+def test_tutorials(dff_tutorial_py_file, virtualenv):
+    with open(dff_tutorial_py_file, "r", encoding="utf-8") as fd:
+        source_code = fd.read()
+
+    check_tutorial_dependencies(
+        virtualenv,
+        source_code,
+    )

tutorials/context_storages/1_basics.py

@@ -5,6 +5,7 @@
 The following tutorial shows the basic use of the database connection.
 """
 
+# %pip install dff[json,pickle]
 
 # %%
 import pathlib

tutorials/context_storages/2_postgresql.py

@@ -5,6 +5,7 @@
 This is a tutorial on using PostgreSQL.
 """
 
+# %pip install dff[postgresql]
 
 # %%
 import os

tutorials/context_storages/3_mongodb.py

@@ -5,6 +5,7 @@
 This is a tutorial on using MongoDB.
 """
 
+# %pip install dff[mongodb]
 
 # %%
 import os

tutorials/context_storages/4_redis.py

@@ -5,6 +5,7 @@
 This is a tutorial on using Redis.
 """
 
+# %pip install dff[redis]
 
 # %%
 import os

tutorials/context_storages/5_mysql.py

@@ -5,6 +5,7 @@
 This is a tutorial on using MySQL.
 """
 
+# %pip install dff[mysql]
 
 # %%
 import os

tutorials/context_storages/6_sqlite.py

@@ -5,6 +5,7 @@
 This is a tutorial on using SQLite.
 """
 
+# %pip install dff[sqlite]
 
 # %%
 import pathlib

tutorials/context_storages/7_yandex_database.py

@@ -5,6 +5,7 @@
 This is a tutorial on how to use Yandex DataBase.
 """
 
+# %pip install dff[ydb]
 
 # %%
 import os