Documentation links in tutorials (#136)

* Add references to API ref in tutorials
* Add another tutorial template (`%mddoclink`)
* Display private fields in API ref

docs/source/conf.py

@@ -146,7 +146,7 @@
 ]
 
 
-autodoc_default_options = {"members": True, "undoc-members": False, "private-members": False}
+autodoc_default_options = {"members": True, "undoc-members": False, "private-members": True}
 
 
 def setup(_):

docs/source/utils/notebook.py

@@ -77,17 +77,22 @@ class DocumentationLink(ReplacePattern):
     USAGE EXAMPLES
     --------------
 
-    [link](%doclink(api,script.core.script))
+    %doclink(api,index_pipeline)) -> ../apiref/index_pipeline.rst
 
-    [link](%doclink(api,script.core.script,Node))
+    %doclink(api,script.core.script) -> ../apiref/dff.script.core.script.rst
 
-    [link](%doclink(tutorial,messengers.web_api_interface.4_streamlit_chat))
+    %doclink(api,script.core.script,Node) -> ../apiref/dff.script.core.script.rst#dff.script.core.script.Node
 
-    [link](%doclink(tutorial,messengers.web_api_interface.4_streamlit_chat,API-configuration))
+    %doclink(tutorial,messengers.web_api_interface.4_streamlit_chat)) ->
+    ../tutorials/tutorials.messengers.web_api_interface.4_streamlit_chat.py
 
-    [link](%doclink(guide,basic_conceptions))
+    %doclink(tutorial,messengers.web_api_interface.4_streamlit_chat,API-configuration) ->
+    ../tutorials/tutorials.messengers.web_api_interface.4_streamlit_chat.py#API-configuration
 
-    [link](%doclink(guide,basic_conceptions,example-conversational-chat-bot))
+    %doclink(guide,basic_conceptions) -> ../user_guides/basic_conceptions.rst
+
+    %doclink(guide,basic_conceptions,example-conversational-chat-bot) ->
+    ../user_guides/basic_conceptions.rst#example-conversational-chat-bot
 
     """
 
@@ -117,6 +122,9 @@ def link_to_doc_page(
             To link to the basic script tutorial, pass "script.core.1_basics" (without the "tutorials" prefix).
 
             To link to the basic concepts guide, pass "basic_conceptions".
+
+            API index pages are also supported.
+            Passing "index_pipeline" will link to the "apiref/index_pipeline.html" page.
         :param anchor:
             An anchor on the page. (optional)
 
@@ -129,7 +137,8 @@ def link_to_doc_page(
             A link to the corresponding documentation part.
         """
         if page_type == "api":
-            return f"../apiref/dff.{page}.rst" + (f"#dff.{page}.{anchor}" if anchor is not None else "")
+            prefix = "" if page.startswith("index") else "dff."
+            return f"../apiref/{prefix}{page}.rst" + (f"#{prefix}{page}.{anchor}" if anchor is not None else "")
         elif page_type == "tutorial":
             return f"../tutorials/tutorials.{page}.py" + (f"#{anchor}" if anchor is not None else "")
         elif page_type == "guide":
@@ -141,8 +150,58 @@ def replacement_string(matchobj: re.Match) -> str:
         return DocumentationLink.link_to_doc_page(*args)
 
 
+class MarkdownDocumentationLink(DocumentationLink):
+    """
+    Replace documentation linking directives with markdown-style links.
+
+    Replace strings of the `%mddoclink({args})` format with corresponding links to local files.
+
+    `args` is a comma-separated string of arguments to pass to the :py:meth:`.DocumentationLink.link_to_doc_page`.
+
+    So, `%mddoclink(arg1,arg2,arg3)` will be replaced with `[text](link_to_doc_page(arg1, arg2, arg3))`, and
+    `%doclink(arg1,arg2)` will be replaced with `[text](link_to_doc_page(arg1, arg2))` with `text` being the last
+    path segment of the last argument.
+
+    USAGE EXAMPLES
+    --------------
+
+    %mddoclink(api,index_pipeline) -> [index_pipeline](
+        ../apiref/index_pipeline.rst
+    )
+
+    %mddoclink(api,script.core.script,Node) -> [Node](
+        ../apiref/dff.script.core.script.rst#dff.script.core.script.Node
+    )
+
+    %mddoclink(tutorial,messengers.web_api_interface.4_streamlit_chat) -> [4_streamlit_chat](
+        ../tutorials/tutorials.messengers.web_api_interface.4_streamlit_chat.py
+    )
+
+    %mddoclink(tutorial,messengers.web_api_interface.4_streamlit_chat,API-configuration) -> [API-configuration](
+        ../tutorials/tutorials.messengers.web_api_interface.4_streamlit_chat.py#API-configuration
+    )
+
+    %mddoclink(guide,basic_conceptions) -> [basic_conceptions](
+        ../user_guides/basic_conceptions.rst
+    )
+
+    %mddoclink(guide,basic_conceptions,example-conversational-chat-bot) -> [example-conversational-chat-bot](
+        ../user_guides/basic_conceptions.rst#example-conversational-chat-bot
+    )
+
+    """
+
+    pattern: ClassVar[re.Pattern] = re.compile(r"%mddoclink\((.+?)\)")
+
+    @staticmethod
+    def replacement_string(matchobj: re.Match) -> str:
+        args = matchobj.group(1).split(",")
+        link_text = args[-1].split(".")[-1]
+        return f"[{link_text}]({DocumentationLink.link_to_doc_page(*args)})"
+
+
 def apply_replace_patterns(text: str) -> str:
-    for cls in (InstallationCell, DocumentationLink):
+    for cls in (InstallationCell, DocumentationLink, MarkdownDocumentationLink):
         text = cls.replace(text)
 
     return text

tests/tutorials/test_format.py

@@ -13,7 +13,7 @@
     re.compile(r"# %%\n"),  # check python block
 ]
 
-docstring_start_pattern = re.compile(r'# %% \[markdown\]\n"""\n#(?: .*:)? \d+\. .*\n(?:\n[\S\s]*)?"""\n')
+docstring_start_pattern = re.compile(r'# %% \[markdown\]\n"""\n#(?: .*:)? \d+\. .*\n(?:\n[\S\s]*)?"""(?:  # .*)?\n')
 comment_start_pattern = re.compile(r'# %% \[markdown\]\n# #(?: .*:)? \d+\. .*\n#(?:\n# [\S\s]*)?')
 
 

tutorials/context_storages/1_basics.py

@@ -3,6 +3,11 @@
 # 1. Basics
 
 The following tutorial shows the basic use of the database connection.
+
+See %mddoclink(api,context_storages.database,context_storage_factory) function
+for creating a context storage by path.
+
+In this example JSON file is used as a storage.
 """
 
 # %pip install dff[json,pickle]

tutorials/context_storages/2_postgresql.py

@@ -3,6 +3,13 @@
 # 2. PostgreSQL
 
 This is a tutorial on using PostgreSQL.
+
+See %mddoclink(api,context_storages.sql,SQLContextStorage) class
+for storing your users' contexts in SQL databases.
+
+DFF uses [sqlalchemy](https://docs.sqlalchemy.org/en/20/)
+and [asyncpg](https://magicstack.github.io/asyncpg/current/)
+libraries for asynchronous access to PostgreSQL DB.
 """
 
 # %pip install dff[postgresql]

tutorials/context_storages/3_mongodb.py

@@ -3,6 +3,12 @@
 # 3. MongoDB
 
 This is a tutorial on using MongoDB.
+
+See %mddoclink(api,context_storages.mongo,MongoContextStorage) class
+for storing you users' contexts in Mongo database.
+
+DFF uses [motor](https://motor.readthedocs.io/en/stable/)
+library for asynchronous access to MongoDB.
 """
 
 # %pip install dff[mongodb]

tutorials/context_storages/4_redis.py

@@ -3,6 +3,12 @@
 # 4. Redis
 
 This is a tutorial on using Redis.
+
+See %mddoclink(api,context_storages.redis,RedisContextStorage) class
+for storing you users' contexts in Redis database.
+
+DFF uses [redis.asyncio](https://redis.readthedocs.io/en/latest/)
+library for asynchronous access to Redis DB.
 """
 
 # %pip install dff[redis]

tutorials/context_storages/5_mysql.py

@@ -3,6 +3,13 @@
 # 5. MySQL
 
 This is a tutorial on using MySQL.
+
+See %mddoclink(api,context_storages.sql,SQLContextStorage) class
+for storing you users' contexts in SQL databases.
+
+DFF uses [sqlalchemy](https://docs.sqlalchemy.org/en/20/)
+and [asyncmy](https://github.com/long2ice/asyncmy)
+libraries for asynchronous access to MySQL DB.
 """
 
 # %pip install dff[mysql]

tutorials/context_storages/6_sqlite.py

@@ -3,6 +3,15 @@
 # 6. SQLite
 
 This is a tutorial on using SQLite.
+
+See %mddoclink(api,context_storages.sql,SQLContextStorage) class
+for storing you users' contexts in SQL databases.
+
+DFF uses [sqlalchemy](https://docs.sqlalchemy.org/en/20/)
+and [aiosqlite](https://readthedocs.org/projects/aiosqlite/)
+libraries for asynchronous access to SQLite DB.
+
+Note that protocol separator for windows differs from one for linux.
 """
 
 # %pip install dff[sqlite]

tutorials/context_storages/7_yandex_database.py

@@ -3,6 +3,12 @@
 # 7. Yandex DataBase
 
 This is a tutorial on how to use Yandex DataBase.
+
+See %mddoclink(api,context_storages.ydb,YDBContextStorage) class
+for storing you users' contexts in Yandex database.
+
+DFF uses [ydb.aio](https://ydb.tech/en/docs/)
+library for asynchronous access to Yandex DB.
 """
 
 # %pip install dff[ydb]

tutorials/context_storages/8_db_benchmarking.py

@@ -50,34 +50,28 @@
 Benchmark results are saved to files.
 
 For that there exist two functions:
-[benchmark_all](
-%doclink(api,utils.db_benchmark.benchmark,benchmark_all)
-)
+%mddoclink(api,utils.db_benchmark.benchmark,benchmark_all)
 and
-[save_results_to_file](
-%doclink(api,utils.db_benchmark.benchmark,save_results_to_file)
-).
+%mddoclink(api,utils.db_benchmark.benchmark,save_results_to_file).
 
 Note: context storages passed into these functions will be cleared.
 
 ### Configuration
 
 The first one is a higher-level wrapper of the second one.
-The first function accepts [BenchmarkCases](
-%doclink(api,utils.db_benchmark.benchmark,BenchmarkCase)
-) which configure databases that are being benchmark and configurations of the benchmarks.
+The first function accepts
+%mddoclink(api,utils.db_benchmark.benchmark,BenchmarkCase)
+which configure databases that are being benchmark and configurations of the benchmarks.
 The second function accepts only a single URI for the database and several benchmark configurations.
 So, the second function is simpler to use, while the first function allows for more configuration
 (e.g. having different databases benchmarked in a single file).
 
-Both function use [BenchmarkConfig](
-%doclink(api,utils.db_benchmark.benchmark,BenchmarkConfig)
-) to configure benchmark behaviour.
+Both function use
+%mddoclink(api,utils.db_benchmark.benchmark,BenchmarkConfig)
+to configure benchmark behaviour.
 `BenchmarkConfig` is only an interface for benchmark configurations.
 Its most basic implementation is
-[BasicBenchmarkConfig](
-%doclink(api,utils.db_benchmark.basic_config,BasicBenchmarkConfig)
-).
+%mddoclink(api,utils.db_benchmark.basic_config,BasicBenchmarkConfig).
 
 It has several parameters:
 
@@ -125,9 +119,9 @@
 """
 ## Viewing benchmark results
 
-Now that the results are saved to a file you can either view them using [report](
-%doclink(api,utils.db_benchmark.report,report)
-) function or [our streamlit app](
+Now that the results are saved to a file you can either view them using the
+%mddoclink(api,utils.db_benchmark.report,report)
+function or [our streamlit app](
 ../_misc/benchmark_streamlit.py
 ).
 """
@@ -187,31 +181,34 @@
 
 ### Configuration presets
 
-The [dff.utils.db_benchmarks.basic_config](
-%doclink(api,utils.db_benchmark.basic_config)
-) module also includes a dictionary containing configuration presets.
+The
+%mddoclink(api,utils.db_benchmark.basic_config)
+module also includes a dictionary containing configuration presets.
 Those cover various contexts and messages as well as some edge cases.
-
-To use configuration presets, simply pass them to benchmark functions.
 """
 
 # %%
-print(benchmark.basic_configurations.keys())
+print(benchmark.basic_config.basic_configurations.keys())
 
-# benchmark.benchmark_all(
-#     ...,
-#     benchmark_configs=benchmark.basic_configurations
-# )
+# %% [markdown]
+"""
+To use configuration presets, simply pass them to benchmark functions:
+
+    benchmark.benchmark_all(
+        ...,
+        benchmark_configs=benchmark.basic_configurations
+    )
+"""
 
 # %% [markdown]
 """
 ### Custom configuration
 
 If the basic configuration is not enough for you, you can create your own.
 
-To do so, inherit from the [dff.utils.db_benchmark.benchmark.BenchmarkConfig](
-%doclink(api,utils.db_benchmark.benchmark,BenchmarkConfig)
-) class. You need to define three methods:
+To do so, inherit from the
+%mddoclink(api,utils.db_benchmark.benchmark,BenchmarkConfig)
+class. You need to define three methods:
 
 - `get_context` -- method to get initial contexts.
 - `info` -- method for getting display info representing the configuration.

tutorials/messengers/telegram/1_basic.py

@@ -4,6 +4,12 @@
 
 The following tutorial shows how to run a regular DFF script in Telegram.
 It asks users for the '/start' command and then loops in one place.
+
+Here, %mddoclink(api,messengers.telegram.interface,PollingTelegramInterface)
+class and [telebot](https://pytba.readthedocs.io/en/latest/index.html)
+library are used for accessing telegram API in polling mode.
+
+Telegram API token is required to access telegram API.
 """
 
 # %pip install dff[telegram]

tutorials/messengers/telegram/2_buttons.py

@@ -2,10 +2,20 @@
 """
 # Telegram: 2. Buttons
 
-
 This tutorial shows how to display and hide a basic keyboard in Telegram.
+
+Here, %mddoclink(api,messengers.telegram.message,TelegramMessage)
+class is used to represent telegram message,
+%mddoclink(api,messengers.telegram.message,TelegramUI) and
+%mddoclink(api,messengers.telegram.message,RemoveKeyboard)
+classes are used for configuring additional telegram message features.
+
+Different %mddoclink(api,script.core.message,message)
+classes are used for representing different common message features,
+like Attachment, Audio, Button, Image, etc.
 """
 
+
 # %pip install dff[telegram]
 
 # %%

tutorials/messengers/telegram/3_buttons_with_callback.py

@@ -2,11 +2,21 @@
 """
 # Telegram: 3. Buttons with Callback
 
-
 This tutorial demonstrates, how to add an inline keyboard and utilize
 inline queries.
+
+Here, %mddoclink(api,messengers.telegram.message,TelegramMessage)
+class is used to represent telegram message,
+%mddoclink(api,messengers.telegram.message,TelegramUI) and
+%mddoclink(api,messengers.telegram.message,RemoveKeyboard)
+classes are used for configuring additional telegram message features.
+
+Different %mddoclink(api,script.core.message,message)
+classes are used for representing different common message features,
+like Attachment, Audio, Button, Image, etc.
 """
 
+
 # %pip install dff[telegram]
 
 # %%

tutorials/messengers/telegram/4_conditions.py

@@ -4,8 +4,12 @@
 
 This tutorial shows how to process Telegram updates in your script
 and reuse handler triggers from the `pytelegrambotapi` library.
+
+Here, %mddoclink(api,messengers.telegram.messenger,telegram_condition)
+function is used for graph navigation according to Telegram events.
 """
 
+
 # %pip install dff[telegram]
 
 # %%