From 934e24e0dbffa47bea5555c260081fc8742d3a24 Mon Sep 17 00:00:00 2001 From: tammes Date: Mon, 12 Dec 2022 20:27:17 +0100 Subject: [PATCH 1/7] added support for libretranslate --- base-config.yaml | 5 +- translate/bot.py | 11 +++-- translate/provider/abstract.py | 2 + translate/provider/libretranslate.py | 70 ++++++++++++++++++++++++++++ translate/util.py | 6 ++- 5 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 translate/provider/libretranslate.py diff --git a/base-config.yaml b/base-config.yaml index 3cfbb5f..338123a 100644 --- a/base-config.yaml +++ b/base-config.yaml @@ -1,7 +1,10 @@ # Translation provider settings provider: id: google - args: {} + args: { + url: 'translate.example.com', # required if you use libretranslate, else irrelevant + api_key: 'loremipsumdolorsitamen1234' # optional if you use libretranslate, else irrelevant + } auto_translate: - room_id: '!roomid:example.com' main_language: en diff --git a/translate/bot.py b/translate/bot.py index f67cbee..bea999f 100644 --- a/translate/bot.py +++ b/translate/bot.py @@ -38,14 +38,14 @@ class TranslatorBot(Plugin): async def start(self) -> None: await super().start() - self.on_external_config_update() + await self.on_external_config_update() - def on_external_config_update(self) -> None: + async def on_external_config_update(self) -> None: self.translator = None self.config.load_and_update() self.auto_translate = self.config.load_auto_translate() try: - self.translator = self.config.load_translator() + self.translator = await self.config.load_translator() except TranslationProviderError: self.log.exception("Error loading translator") @@ -84,7 +84,8 @@ def is_acceptable(lang: str) -> bool: async def command_handler(self, evt: MessageEvent, language: Optional[Tuple[str, str]], text: str) -> None: if not language: - await evt.reply("Usage: !translate [from] [text or reply to message]") + await evt.reply("No supported target language detected. " + "Usage: !translate [from] [text or reply to message]") return if not self.config["response_reply"]: evt.disable_reply = True @@ -95,7 +96,7 @@ async def command_handler(self, evt: MessageEvent, language: Optional[Tuple[str, reply_evt = await self.client.get_event(evt.room_id, evt.content.get_reply_to()) text = reply_evt.content.body if not text: - await evt.reply("Usage: !translate [from] [text or reply to message]") + await evt.reply("Nothing to translate detected. Usage: !translate [from] [text or reply to message]") return result = await self.translator.translate(text, to_lang=language[1], from_lang=language[0]) await evt.reply(result.text) diff --git a/translate/provider/abstract.py b/translate/provider/abstract.py index e5839be..f5a64d5 100644 --- a/translate/provider/abstract.py +++ b/translate/provider/abstract.py @@ -23,6 +23,8 @@ class AbstractTranslationProvider(ABC): @abstractmethod def __init__(self, args: Dict) -> None: pass + async def post_init(self) -> None: + pass @abstractmethod async def translate(self, text: str, to_lang: str, from_lang: str = "auto") -> Result: diff --git a/translate/provider/libretranslate.py b/translate/provider/libretranslate.py new file mode 100644 index 0000000..246861f --- /dev/null +++ b/translate/provider/libretranslate.py @@ -0,0 +1,70 @@ +# translate - A maubot plugin to translate words. +# Copyright (C) 2022 Tammes Burghard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import json +import re +from typing import Dict + +from aiohttp import ClientSession, client_exceptions +from yarl import URL + +from . import AbstractTranslationProvider, Result + + +class LibreTranslate(AbstractTranslationProvider): + headers: Dict[str, str] = {"Content-Type": "application/json"} + + def __init__(self, args: Dict) -> None: + super().__init__(args) + if args.get("url") is None: + raise ValueError("Please specify the url of your preferred libretranslate instance in provider.args.url") + else: + self._base_url = args.get("url") + if not re.match(r"^https?://", self._base_url): + self._base_url = "https://" + self._base_url + self.url: URL = URL(self._base_url + "/translate") + self.api_key = args.get("api_key") + self.supported_languages = None + + async def post_init(self): + try: + async with ClientSession() as sess: + resp = await sess.get(self._base_url + "/languages") + self.supported_languages = {lang["code"]: lang["name"] for lang in await resp.json()} + except client_exceptions.ClientError: + raise ValueError(f"This url ({self._base_url}) does not point to a compatible libretranslate instance. " + f"Please change it") + + async def translate(self, text: str, to_lang: str, from_lang: str = "auto") -> Result: + if not from_lang: + from_lang = "auto" + async with ClientSession() as sess: + data=json.dumps({"q": text, "source": from_lang, "target": to_lang, + "format": "text", "api_key": self.api_key}) + resp = await sess.post(self.url, data=data, headers=self.headers) + if resp.status == 403: + raise ValueError("Request forbidden. You did probably configure an incorrect api key.") + data = await resp.json() + return Result(text=data["translatedText"], + source_language=data["detectedLanguage"]["language"] if from_lang == "auto" else from_lang) + + def is_supported_language(self, code: str) -> bool: + return code.lower() in self.supported_languages.keys() + + def get_language_name(self, code: str) -> str: + return self.supported_languages[code] + + +make_translation_provider = LibreTranslate diff --git a/translate/util.py b/translate/util.py index 438b459..34dcf0d 100644 --- a/translate/util.py +++ b/translate/util.py @@ -41,7 +41,7 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: helper.copy("auto_translate") helper.copy("response_reply") - def load_translator(self) -> AbstractTranslationProvider: + async def load_translator(self) -> AbstractTranslationProvider: try: provider = self["provider.id"] mod = import_module(f".{provider}", "translate.provider") @@ -49,7 +49,9 @@ def load_translator(self) -> AbstractTranslationProvider: except (KeyError, AttributeError, ImportError) as e: raise TranslationProviderError("Failed to load translation provider") from e try: - return make(self["provider.args"]) + translation_provider = make(self["provider.args"]) + await translation_provider.post_init() + return translation_provider except Exception as e: raise TranslationProviderError("Failed to initialize translation provider") from e From 6a89c70c376cf5c4e4108221f4835655aa0b550c Mon Sep 17 00:00:00 2001 From: tammes Date: Mon, 12 Dec 2022 20:35:44 +0100 Subject: [PATCH 2/7] Updated README.md --- README.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2d8dd2a..e37998c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # translate -A [maubot](https://github.com/maubot/maubot) to translate words using Google Translate (DeepL is planned too) +A [maubot](https://github.com/maubot/maubot) to translate words using Google Translate, DeepL or LibreTranslate ## Usage @@ -19,15 +19,11 @@ You can also use the alias `tr`: !tr en ru Hello world. The first parameter (source language) can be set to `auto` or omitted entirely -to let Google Translate detect the source language. Additionally, you can reply +to let the bot detect the source language. Additionally, you can reply to a message with `!tr ` (no text) to translate the message you replied to. ## supported languages: -- de: (german) -- en: (english) -- zh: (chinese) -- ... - -Full list of supported languages: https://cloud.google.com/translate/docs/languages +This depends on the translation provider you choose. For google, a list of +supported languages can be found at https://cloud.google.com/translate/docs/languages From f95066cc8e701e816a4e389f6a16d5c3cd4da1f2 Mon Sep 17 00:00:00 2001 From: tammes Date: Thu, 15 Dec 2022 17:20:55 +0100 Subject: [PATCH 3/7] correct supported translation providers --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e37998c..31b9069 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # translate -A [maubot](https://github.com/maubot/maubot) to translate words using Google Translate, DeepL or LibreTranslate +A [maubot](https://github.com/maubot/maubot) to translate words using Google Translate +or LibreTranslate (Deepl is also planned) ## Usage From 217bf5d2206b5bee1751ad17e807f6cf2e2c2750 Mon Sep 17 00:00:00 2001 From: tammes Date: Thu, 15 Dec 2022 18:11:49 +0100 Subject: [PATCH 4/7] add !languages command --- translate/bot.py | 6 ++++++ translate/provider/abstract.py | 3 +++ translate/provider/deepl.py | 4 ++++ translate/provider/google.py | 4 ++++ translate/provider/libretranslate.py | 8 ++++++-- 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/translate/bot.py b/translate/bot.py index bea999f..3472c70 100644 --- a/translate/bot.py +++ b/translate/bot.py @@ -100,3 +100,9 @@ async def command_handler(self, evt: MessageEvent, language: Optional[Tuple[str, return result = await self.translator.translate(text, to_lang=language[1], from_lang=language[0]) await evt.reply(result.text) + + @command.new("languages") + async def languages_command_handler(self, evt: MessageEvent) -> None: + await evt.reply( + ", ".join([f"{name} `{code}`" for code, name in self.translator.get_supported_languages().items()]) + ) diff --git a/translate/provider/abstract.py b/translate/provider/abstract.py index f5a64d5..f4ba526 100644 --- a/translate/provider/abstract.py +++ b/translate/provider/abstract.py @@ -37,3 +37,6 @@ def is_supported_language(self, code: str) -> bool: @abstractmethod def get_language_name(self, code: str) -> str: pass + @abstractmethod + def get_supported_languages(self) -> dict: + pass diff --git a/translate/provider/deepl.py b/translate/provider/deepl.py index 02ab8c9..58cfdf8 100644 --- a/translate/provider/deepl.py +++ b/translate/provider/deepl.py @@ -126,5 +126,9 @@ def is_supported_language(self, code: str) -> bool: def get_language_name(self, code: str) -> str: return self.supported_languages[code] + def get_supported_languages(self) -> dict: + return self.supported_languages + + make_translation_provider = DeepLTranslate diff --git a/translate/provider/google.py b/translate/provider/google.py index 7923bca..9b5fa5d 100644 --- a/translate/provider/google.py +++ b/translate/provider/google.py @@ -76,5 +76,9 @@ def is_supported_language(self, code: str) -> bool: def get_language_name(self, code: str) -> str: return self.supported_languages[code] + def get_supported_languages(self) -> dict: + return self.supported_languages + + make_translation_provider = GoogleTranslate diff --git a/translate/provider/libretranslate.py b/translate/provider/libretranslate.py index 246861f..5fbd1be 100644 --- a/translate/provider/libretranslate.py +++ b/translate/provider/libretranslate.py @@ -36,13 +36,14 @@ def __init__(self, args: Dict) -> None: self._base_url = "https://" + self._base_url self.url: URL = URL(self._base_url + "/translate") self.api_key = args.get("api_key") - self.supported_languages = None + self.supported_languages = {"auto": "Detect language"} async def post_init(self): try: async with ClientSession() as sess: resp = await sess.get(self._base_url + "/languages") - self.supported_languages = {lang["code"]: lang["name"] for lang in await resp.json()} + for lang in await resp.json(): + self.supported_languages[lang["code"]] = lang["name"] except client_exceptions.ClientError: raise ValueError(f"This url ({self._base_url}) does not point to a compatible libretranslate instance. " f"Please change it") @@ -66,5 +67,8 @@ def is_supported_language(self, code: str) -> bool: def get_language_name(self, code: str) -> str: return self.supported_languages[code] + def get_supported_languages(self) -> dict: + return self.supported_languages + make_translation_provider = LibreTranslate From 14eae8acc392d9276edac94b987c952f996c83d7 Mon Sep 17 00:00:00 2001 From: tammes Date: Thu, 15 Dec 2022 18:22:51 +0100 Subject: [PATCH 5/7] remove abundant blank lines --- translate/provider/deepl.py | 1 - translate/provider/google.py | 1 - 2 files changed, 2 deletions(-) diff --git a/translate/provider/deepl.py b/translate/provider/deepl.py index 58cfdf8..39e600e 100644 --- a/translate/provider/deepl.py +++ b/translate/provider/deepl.py @@ -130,5 +130,4 @@ def get_supported_languages(self) -> dict: return self.supported_languages - make_translation_provider = DeepLTranslate diff --git a/translate/provider/google.py b/translate/provider/google.py index 9b5fa5d..6325dab 100644 --- a/translate/provider/google.py +++ b/translate/provider/google.py @@ -80,5 +80,4 @@ def get_supported_languages(self) -> dict: return self.supported_languages - make_translation_provider = GoogleTranslate From cac4322f0c097315bc89ad06e3e1fb2d1f88596a Mon Sep 17 00:00:00 2001 From: tammes Date: Tue, 20 Dec 2022 11:35:18 +0100 Subject: [PATCH 6/7] add !languages command to README.md --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 31b9069..dcd5eb9 100644 --- a/README.md +++ b/README.md @@ -26,5 +26,8 @@ replied to. ## supported languages: -This depends on the translation provider you choose. For google, a list of -supported languages can be found at https://cloud.google.com/translate/docs/languages +Since this depends on which translation provider you chose and can even vary from +one LibreTranslate instance to another, you can use the following command to get +all the supported languages and their codes: + + !languages From a5f1062f9a382d4158f40a8ada7f111a4eb993f4 Mon Sep 17 00:00:00 2001 From: tammes Date: Tue, 20 Dec 2022 11:48:40 +0100 Subject: [PATCH 7/7] add hyperlinks for translation providers to README.md --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dcd5eb9..af20462 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # translate -A [maubot](https://github.com/maubot/maubot) to translate words using Google Translate -or LibreTranslate (Deepl is also planned) +A [maubot](https://github.com/maubot/maubot) to translate words using [Google +Translate](https://translate.google.com/about/) or +[LibreTranslate](https://github.com/LibreTranslate/LibreTranslate) +(Deepl is planned too) ## Usage @@ -26,7 +28,7 @@ replied to. ## supported languages: -Since this depends on which translation provider you chose and can even vary from +Since this depends on which translation provider you choose and can even vary from one LibreTranslate instance to another, you can use the following command to get all the supported languages and their codes: