From 3a84628aa3091aa08dee35b2e3c520e6919b2ab0 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Thu, 22 Feb 2024 11:45:08 -0500 Subject: [PATCH 01/18] added sonarcloud --- .DS_Store | Bin 0 -> 8196 bytes README.md | 5 ++--- classes/output/glossary_page.php | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..4c0c4551059203053dd557da182beff022a8b9a7 GIT binary patch literal 8196 zcmeHMTWl3Y7@lui=WdnKw^tSQL8CknA56p-A53G!1bi^@MISWr2~G5$ojs&2^u-WE!cH>t&;QT- zvoqg(|IV2`j4`wpt%n$^V~laiLh3W9xJ44}BHxsh*iuOnV5Wr9G0z+H)ect4~m3z>=KsFcyGgNo=1KvG-* zqC|CFR7j>{nTh47l+;2A(v_riMSR78bSHZ{?w5(>sFc#3A-;Sdelp@43Zkde`01!S zLq^J^Ob|#AxETQvK9$U3KFhOU{QW&Q>^ec%-26S1ippuzt9TWk!H;wf`Xim885V<9 zvv5dgUB2(;%5^i>v)zw(&`sID~NnMWdm2an$_#h zoN3vxrYXI)rR8i>`pm}F%~W5r{_I(m&uv(>`C#^>JLLH<$R!|t4X|00mh|HI^|(Q) zO@5E4s4lPPhw^$ontFX*ug0m-aBC!odc&imehf-Ty6^S)ag z@`9{hoj2Wl(KSPRyX$s3PuM|UXrBs2Z6NRqVO!L(&mmu7%j$YDun*eg;FRngssMTmvyF7Eq9-BA6PB#`9_fF--^#y1a+A38K)UK1Xz0!5HwwCLb+U=S-+m)sb z+D5MLReMVkUumq-9_DIa-X9vG#RSrpFUjFQ?`+QBk(Cu;tzL{5NOF_qF4iZQ9KT$R%F3W^{#@?FzNDq>|ri>B$76ic#g%R*9FtYo&+Fd{Pk z14ln$7ulEWG8-d~{=$A`zq5acm$OkzoLq`!#LFhEAYQg$6EdNg&y>w z9|Oq4#t96F#MqDV#Uu{yp2$I59M8d0 zXTfuQ?*!S(oSoP`(QY#W6>_{fasO|h`uqQFerhsYfDwt|K$9S_;sitelang Default Moodle Language + * @var string $sitelang Default Moodle Language */ private string $sitelang; From a81cb9127bc044b29e4f9ae9ae59287751768011 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Thu, 22 Feb 2024 11:45:41 -0500 Subject: [PATCH 02/18] updated gitignore --- .DS_Store | Bin 8196 -> 0 bytes .gitignore | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 4c0c4551059203053dd557da182beff022a8b9a7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMTWl3Y7@lui=WdnKw^tSQL8CknA56p-A53G!1bi^@MISWr2~G5$ojs&2^u-WE!cH>t&;QT- zvoqg(|IV2`j4`wpt%n$^V~laiLh3W9xJ44}BHxsh*iuOnV5Wr9G0z+H)ect4~m3z>=KsFcyGgNo=1KvG-* zqC|CFR7j>{nTh47l+;2A(v_riMSR78bSHZ{?w5(>sFc#3A-;Sdelp@43Zkde`01!S zLq^J^Ob|#AxETQvK9$U3KFhOU{QW&Q>^ec%-26S1ippuzt9TWk!H;wf`Xim885V<9 zvv5dgUB2(;%5^i>v)zw(&`sID~NnMWdm2an$_#h zoN3vxrYXI)rR8i>`pm}F%~W5r{_I(m&uv(>`C#^>JLLH<$R!|t4X|00mh|HI^|(Q) zO@5E4s4lPPhw^$ontFX*ug0m-aBC!odc&imehf-Ty6^S)ag z@`9{hoj2Wl(KSPRyX$s3PuM|UXrBs2Z6NRqVO!L(&mmu7%j$YDun*eg;FRngssMTmvyF7Eq9-BA6PB#`9_fF--^#y1a+A38K)UK1Xz0!5HwwCLb+U=S-+m)sb z+D5MLReMVkUumq-9_DIa-X9vG#RSrpFUjFQ?`+QBk(Cu;tzL{5NOF_qF4iZQ9KT$R%F3W^{#@?FzNDq>|ri>B$76ic#g%R*9FtYo&+Fd{Pk z14ln$7ulEWG8-d~{=$A`zq5acm$OkzoLq`!#LFhEAYQg$6EdNg&y>w z9|Oq4#t96F#MqDV#Uu{yp2$I59M8d0 zXTfuQ?*!S(oSoP`(QY#W6>_{fasO|h`uqQFerhsYfDwt|K$9S_; Date: Fri, 23 Feb 2024 12:46:26 -0500 Subject: [PATCH 03/18] removed source lang option --- classes/output/glossary_page.php | 8 +++----- classes/output/manage_page.php | 9 +++------ templates/glossary_page.mustache | 4 ++-- templates/manage_page.mustache | 8 ++++---- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/classes/output/glossary_page.php b/classes/output/glossary_page.php index 54b6e2b..344ccf9 100644 --- a/classes/output/glossary_page.php +++ b/classes/output/glossary_page.php @@ -46,12 +46,12 @@ class glossary_page implements renderable, templatable { private array $langs; /** - * @var string $source_lange Source language of the text + * @var string $sourcelang Source language of the text */ private string $sourcelang; /** - * @var string $target_lang Target language of the text + * @var string $targetlang Target language of the text */ private string $targetlang; @@ -114,8 +114,7 @@ public function __construct() { // Qury params. $this->sitelang = get_config('core', 'lang', PARAM_NOTAGS); $this->langs = $translator->getsupportedglossarylangs(); - $this->sourcelang = optional_param('source_lang', $this->sitelang, PARAM_NOTAGS); - $this->sourcelang = clean_param($this->sourcelang, PARAM_NOTAGS); + $this->sourcelang = $this->sitelang; $this->targetlang = optional_param('target_lang', $this->sitelang, PARAM_NOTAGS); $this->targetlang = clean_param($this->targetlang, PARAM_NOTAGS); $this->page = optional_param('page', 1, PARAM_INT); @@ -135,7 +134,6 @@ public function __construct() { // Url params. $urlparams = []; - $urlparams['source_lang'] = $this->sourcelang; $urlparams['target_lang'] = $this->targetlang; $urlparams['limit'] = $this->limit; $urlparams['page'] = $this->page; diff --git a/classes/output/manage_page.php b/classes/output/manage_page.php index a10e97a..9df53bd 100644 --- a/classes/output/manage_page.php +++ b/classes/output/manage_page.php @@ -48,14 +48,13 @@ class manage_page implements renderable, templatable { */ private array $langs; - /** - * @var string $source_lange Source language of the text + * @var string $sourcelang Target language of the text */ private string $sourcelang; /** - * @var string $target_lang Target language of the text + * @var string $targetlang Target language of the text */ private string $targetlang; @@ -126,8 +125,7 @@ public function __construct() { // Qury params. $this->sitelang = get_config('core', 'lang', PARAM_NOTAGS); $this->langs = get_string_manager()->get_list_of_translations(); - $this->sourcelang = optional_param('source_lang', $this->sitelang, PARAM_NOTAGS); - $this->sourcelang = clean_param($this->sourcelang, PARAM_NOTAGS); + $this->sourcelang = $this->sitelang; $this->targetlang = optional_param('target_lang', $this->sitelang, PARAM_NOTAGS); $this->targetlang = clean_param($this->targetlang, PARAM_NOTAGS); $this->instanceid = optional_param('instanceid', null, PARAM_INT); @@ -152,7 +150,6 @@ public function __construct() { // Url params. $urlparams = []; - $urlparams['source_lang'] = $this->sourcelang; $urlparams['target_lang'] = $this->targetlang; $urlparams['limit'] = $this->limit; $urlparams['page'] = $this->page; diff --git a/templates/glossary_page.mustache b/templates/glossary_page.mustache index 1b37378..92fec9d 100644 --- a/templates/glossary_page.mustache +++ b/templates/glossary_page.mustache @@ -64,14 +64,14 @@ along with Moodle. If not, see .

{{#str}}site_language,filter_autotranslate{{/str}}

{{#str}}supported_glossary_langs,filter_autotranslate{{/str}}

diff --git a/templates/manage_page.mustache b/templates/manage_page.mustache index 256fd47..e4520c0 100644 --- a/templates/manage_page.mustache +++ b/templates/manage_page.mustache @@ -58,13 +58,13 @@ along with Moodle. If not, see .
@@ -78,7 +78,7 @@ along with Moodle. If not, see .

{{#str}}select_target_language,filter_autotranslate{{/str}}

From a4b034c893366a0616c8bdcb4e8fc6119dbf3993 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Fri, 23 Feb 2024 12:52:17 -0500 Subject: [PATCH 04/18] removed source_lang from forms --- classes/output/glossary_form.php | 1 - classes/output/manage_form.php | 1 - 2 files changed, 2 deletions(-) diff --git a/classes/output/glossary_form.php b/classes/output/glossary_form.php index cbfb54b..5d059bd 100644 --- a/classes/output/glossary_form.php +++ b/classes/output/glossary_form.php @@ -60,7 +60,6 @@ public function definition() { // URL params for pagination. $this->urlparams = []; - $this->urlparams['source_lang'] = $sourcelang; $this->urlparams['target_lang'] = $targetlang; $this->urlparams['limit'] = $limit; diff --git a/classes/output/manage_form.php b/classes/output/manage_form.php index 7e16c00..0f132b2 100644 --- a/classes/output/manage_form.php +++ b/classes/output/manage_form.php @@ -63,7 +63,6 @@ public function definition() { // URL params for pagination. $this->urlparams = []; - $this->urlparams['source_lang'] = $sourcelang; $this->urlparams['target_lang'] = $targetlang; $this->urlparams['limit'] = $limit; if ($instanceid) { From a458254f1782c987861bd7493c2f3f717cd5ad58 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Fri, 23 Feb 2024 12:53:24 -0500 Subject: [PATCH 05/18] removed source_lang from forms --- lib.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib.php b/lib.php index ec245bc..10faf40 100644 --- a/lib.php +++ b/lib.php @@ -38,7 +38,7 @@ function filter_autotranslate_extend_navigation_course($navigation, $course) { // Build a moodle url. $url = new moodle_url( - "/filter/autotranslate/manage.php?sourcelang=$sitelang&targetlang=$currentlang&limit=500&instanceid=$course->id" + "/filter/autotranslate/manage.php?targetlang=$currentlang&limit=500&instanceid=$course->id" ); // Get title of translate page for navigation menu. From 8e757cb447dad8a18f5ebb1842f69229ad0b8071 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Mon, 26 Feb 2024 21:25:49 -0500 Subject: [PATCH 06/18] fixed ready field in xml --- db/install.xml | 2 +- lib.php | 32 +++++++++++++++++++++++++------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/db/install.xml b/db/install.xml index dd04af3..52a1938 100644 --- a/db/install.xml +++ b/db/install.xml @@ -87,7 +87,7 @@ - + diff --git a/lib.php b/lib.php index 10faf40..7237644 100644 --- a/lib.php +++ b/lib.php @@ -37,20 +37,38 @@ function filter_autotranslate_extend_navigation_course($navigation, $course) { $currentlang = current_language(); // Build a moodle url. - $url = new moodle_url( + $manageurl = new moodle_url( "/filter/autotranslate/manage.php?targetlang=$currentlang&limit=500&instanceid=$course->id" ); // Get title of translate page for navigation menu. - $title = get_string('manage_title', 'filter_autotranslate'); + $managetitle = get_string('manage_title', 'filter_autotranslate'); // Navigation node. - $translatecontent = navigation_node::create( - $title, - $url, + $managecontent = navigation_node::create( + $managetitle, + $manageurl, navigation_node::TYPE_CUSTOM, - $title, + $managetitle, 'autotranslate', ); - $navigation->add_node($translatecontent); + $navigation->add_node($managecontent); + + // Build a moodle url. + $glossaryurl = new moodle_url( + "/filter/autotranslate/glossary.php" + ); + + // Get title of glossary page for navigation menu. + $glossarytitle = get_string('glossary_title', 'filter_autotranslate'); + + // Navigation node. + $glossarycontent = navigation_node::create( + $glossarytitle, + $glossaryurl, + navigation_node::TYPE_CUSTOM, + $glossarytitle, + 'autotranslate', + ); + $navigation->add_node($glossarycontent); } From 070910ef09c393bdc259a3ddb000f54df6d0bf16 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Mon, 26 Feb 2024 21:27:40 -0500 Subject: [PATCH 07/18] updated name index --- db/install.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/install.xml b/db/install.xml index 52a1938..a52da5a 100644 --- a/db/install.xml +++ b/db/install.xml @@ -86,7 +86,7 @@ - + From e6849554a44a00656cb2cc7437ca74137b3c40ea Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Mon, 26 Feb 2024 21:39:59 -0500 Subject: [PATCH 08/18] api error --- classes/autotranslate/translator.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/autotranslate/translator.php b/classes/autotranslate/translator.php index c331d6c..a59cd44 100644 --- a/classes/autotranslate/translator.php +++ b/classes/autotranslate/translator.php @@ -66,7 +66,7 @@ public function __construct() { // Get the api key from settings. $authkey = get_config('filter_autotranslate', 'deeplapikey'); if (!$authkey) { - throw new \moodle_exception('missingapikey', 'filter_autotranslate'); + // throw new \moodle_exception('missingapikey', 'filter_autotranslate'); } // Load deepl translator. From 944cd6d09db48f04f8f4812ce72a470237cee874 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Mon, 26 Feb 2024 21:40:46 -0500 Subject: [PATCH 09/18] api error --- classes/autotranslate/translator.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/classes/autotranslate/translator.php b/classes/autotranslate/translator.php index a59cd44..a7711c0 100644 --- a/classes/autotranslate/translator.php +++ b/classes/autotranslate/translator.php @@ -66,7 +66,10 @@ public function __construct() { // Get the api key from settings. $authkey = get_config('filter_autotranslate', 'deeplapikey'); if (!$authkey) { - // throw new \moodle_exception('missingapikey', 'filter_autotranslate'); + $this->translator = null; + } else { + // Load deepl translator. + $this->translator = new \DeepL\Translator($authkey); } // Load deepl translator. From 489d98f047955eac1d6330ccae328619b7616758 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Mon, 26 Feb 2024 21:43:54 -0500 Subject: [PATCH 10/18] api error --- classes/autotranslate/translator.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/autotranslate/translator.php b/classes/autotranslate/translator.php index a7711c0..2fc7e73 100644 --- a/classes/autotranslate/translator.php +++ b/classes/autotranslate/translator.php @@ -36,7 +36,7 @@ class translator { /** * @var \DeepL\Translator $translator DeepL Translator */ - public \DeepL\Translator $translator; + public ?\DeepL\Translator $translator; /** * @var array $sourcelangs Supported Source Languages From 9286843ce9568a527ed06d29656bffefdac7ad28 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Mon, 26 Feb 2024 21:44:43 -0500 Subject: [PATCH 11/18] api error --- classes/autotranslate/translator.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/classes/autotranslate/translator.php b/classes/autotranslate/translator.php index 2fc7e73..9df9d29 100644 --- a/classes/autotranslate/translator.php +++ b/classes/autotranslate/translator.php @@ -72,8 +72,7 @@ public function __construct() { $this->translator = new \DeepL\Translator($authkey); } - // Load deepl translator. - $this->translator = new \DeepL\Translator($authkey); + // Get the langs supported by the site. $this->langs = get_string_manager()->get_list_of_translations(); } From 466ffe30cf1bfe7d5ca9f7f680d844d801f2f261 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Mon, 26 Feb 2024 21:45:45 -0500 Subject: [PATCH 12/18] api error --- classes/autotranslate/translator.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/classes/autotranslate/translator.php b/classes/autotranslate/translator.php index 9df9d29..d747b36 100644 --- a/classes/autotranslate/translator.php +++ b/classes/autotranslate/translator.php @@ -136,13 +136,13 @@ public function getsupportedglossarylangs() { * Get DeepL Usage */ public function getusage() { - return $this->translator->getUsage(); + return $this->translator ? $this->translator->getUsage() : 0; } /** * Get Glossaries. */ public function listglossaries() { - return $this->translator->listGlossaries(); + return $this->translator ? $this->translator->listGlossaries() : []; } } From d3bbe2843498dd2ca9ce52e4085a38aff6c664e7 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Tue, 27 Feb 2024 11:19:54 -0500 Subject: [PATCH 13/18] updated readme requirements --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 98a4508..3f7ecac 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,14 @@ [![Latest Release](https://img.shields.io/github/v/release/jamfire/moodle-filter_autotranslate)](https://github.com/jamfire/moodle-filter_autotranslate/releases) [![Moodle Plugin CI](https://github.com/jamfire/moodle-filter_autotranslate/actions/workflows/moodle-ci.yml/badge.svg)](https://github.com/jamfire/moodle-filter_autotranslate/actions/workflows/moodle-ci.yml) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=jamfire_moodle-filter_autotranslate&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=jamfire_moodle-filter_autotranslate) +## Requirements + +- PHP 8.2+ +- Moodle 4.2+ + ## Installation -- Unzip the plugin in the moodle .../filter/ directory. +- Unzip the plugin in the moodle .../filter/ directory. This plugin is in an Alpha state and it is highly recommended that you backup your database before installing and enabling this plugin. @@ -46,8 +51,8 @@ You can signup for a free or pro version key of DeepL's [API.](https://www.deepl ## Enable the filter and move it to the top -- Go to "Site Administration >> Plugins >> Filters >> Manage filters" and enable the plugin there. -- It is recommended that you position the Autotranslate Filter at the top of your filter list and enable it on headings and content. +- Go to "Site Administration >> Plugins >> Filters >> Manage filters" and enable the plugin there. +- It is recommended that you position the Autotranslate Filter at the top of your filter list and enable it on headings and content. ## Autotranslation scheduled task From 4930dea819cfe577fe1c2f7839e64e4607892807 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Tue, 27 Feb 2024 18:36:46 -0500 Subject: [PATCH 14/18] refactor the filter to detect translations from core --- classes/task/autotranslate_task.php | 18 ++--- classes/task/check_source_task.php | 111 ++++++++++++++++++++++++++++ db/install.xml | 7 +- db/tasks.php | 9 +++ db/upgrade.php | 73 ++++++++++++++++++ filter.php | 61 +++++++++------ lang/en/filter_autotranslate.php | 1 + vendor/composer/installed.php | 4 +- version.php | 4 +- 9 files changed, 252 insertions(+), 36 deletions(-) create mode 100644 classes/task/check_source_task.php diff --git a/classes/task/autotranslate_task.php b/classes/task/autotranslate_task.php index 22b4d63..335ee40 100644 --- a/classes/task/autotranslate_task.php +++ b/classes/task/autotranslate_task.php @@ -69,7 +69,7 @@ public function execute() { // Get 100 existing jobs. $jobs = $DB->get_records('filter_autotranslate_jobs', [ 'fetched' => '0', - 'source_missing' => '0', + 'source' => '1', ], null, '*', 0, $fetchlimit); $jobscount = count($jobs); mtrace("$jobscount jobs found..."); @@ -129,24 +129,24 @@ public function execute() { } // Update the job to fetched. - $jid = $DB->update_record( + $DB->set_field( 'filter_autotranslate_jobs', + 'modified_at', + time(), [ 'id' => $job->id, - 'fetched' => "1", ] ); - - mtrace("completed job $job->id..."); - } else if (!$sourcerecord) { - // Update the job to fetched. - $jid = $DB->update_record( + $jid = $DB->set_field( 'filter_autotranslate_jobs', + 'fetched', + '1', [ 'id' => $job->id, - 'source_missing' => "1", ] ); + + mtrace("completed job $job->id..."); } } } diff --git a/classes/task/check_source_task.php b/classes/task/check_source_task.php new file mode 100644 index 0000000..a376b74 --- /dev/null +++ b/classes/task/check_source_task.php @@ -0,0 +1,111 @@ +. + +namespace filter_autotranslate\task; + +/** + * Autotranslate Jobs + * + * Checks against the job database for unfetched translations + * + * @package filter_autotranslate + * @copyright 2024 Kaleb Heitzman + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class check_source_task extends \core\task\scheduled_task { + /** + * Get name of the Check Source Task + */ + public function get_name() { + return get_string('checksourcetask', 'filter_autotranslate'); + } + + /** + * Execute the Autotranslate Task + */ + public function execute() { + global $DB; + + // Get the site language. + $sitelang = get_config('core', 'lang'); + + // Get the fetch limit. + $fetchlimit = get_config('filter_autotranslate', 'fetchlimit'); + if (!$fetchlimit) { + $fetchlimit = 200; + } + + // Your task logic goes here. + mtrace("Executing autotranslation check source tasks..."); + + + // Get 100 existing jobs. + $jobs = $DB->get_records('filter_autotranslate_jobs', [ + 'fetched' => '0', + 'source' => '0', + ], null, '*'); + $jobscount = count($jobs); + mtrace("$jobscount jobs found..."); + + // Iterate through the jos and add translations. + foreach ($jobs as $job) { + mtrace("checking $job->lang source for $job->hash key..."); + + // Get the source text. + $sourcerecord = $DB->get_record('filter_autotranslate', ['hash' => $job->hash, 'lang' => $sitelang], 'id'); + if ($sourcerecord) { + // Update the job to fetched. + mtrace("source for $job->id found..."); + + // Set the source to 1. + $DB->set_field( + 'filter_autotranslate_jobs', + 'source', + '1', + [ + 'id' => $job->id, + ] + ); + + // Set the modified time. + $DB->set_field( + 'filter_autotranslate_jobs', + 'modified_at', + time(), + [ + 'id' => $job->id, + ] + ); + } else { + // No source record found and we should + // see if the record should be deleted. + $currenttime = time(); + $timediff = $currenttime - $job->modified_at; + $timelimit = 60 * 60 * 24 * 7; // 7 days + + if ($timediff > $timelimit) { + mtrace("check source job $job->id for $job->lang has expired, cleaning up..."); + $DB->delete_records( + 'filter_autotranslate_jobs', + [ + 'id' => $job->id, + ] + ); + } + } + } + } +} diff --git a/db/install.xml b/db/install.xml index a52da5a..5a261f4 100644 --- a/db/install.xml +++ b/db/install.xml @@ -30,7 +30,10 @@ - + + + + @@ -38,7 +41,7 @@ - + diff --git a/db/tasks.php b/db/tasks.php index 2684c41..f78671b 100644 --- a/db/tasks.php +++ b/db/tasks.php @@ -25,6 +25,15 @@ defined('MOODLE_INTERNAL') || die(); $tasks = [ + [ + 'classname' => 'filter_autotranslate\task\check_source_task', + 'blocking' => 0, + 'minute' => '*', + 'hour' => '*', + 'day' => '*', + 'month' => '*', + 'dayofweek' => '*', + ], [ 'classname' => 'filter_autotranslate\task\autotranslate_task', 'blocking' => 0, diff --git a/db/upgrade.php b/db/upgrade.php index c4fd779..d4495c2 100644 --- a/db/upgrade.php +++ b/db/upgrade.php @@ -238,5 +238,78 @@ function xmldb_filter_autotranslate_upgrade($oldversion) { upgrade_plugin_savepoint(true, 2024021700, 'filter', 'autotranslate'); } + // Add status field for tracking translations. + if ($oldversion < 2024022700) { + // Define the table. + $table = new xmldb_table('filter_autotranslate_jobs'); + + // Drop the source_missing index on _jobs. + $index = new xmldb_index('source_missing_index', XMLDB_INDEX_NOTUNIQUE, ['source_missing']); + $dbman->drop_index($table, $index); + + // Modify the source_missing field to just source on _jobs. + $field = new xmldb_field('source_missing', XMLDB_TYPE_INTEGER, '1', null, null, null, 0); + $dbman->rename_field($table, $field, 'source'); + + // Add the source index on _jobs. + $index = new xmldb_index('source_index', XMLDB_INDEX_NOTUNIQUE, ['source']); + $dbman->add_index($table, $index); + + // Update the version number to the latest. + upgrade_plugin_savepoint(true, 2024022700, 'filter', 'autotranslate'); + } + + // Update old source_missing field values. + if ($oldversion < 2024022701) { + $tablename = 'filter_autotranslate_jobs'; + $fieldname = 'source'; + $oldvalue = '1'; + $newvalue = '0'; + + // Query records with the old value. + $records = $DB->get_records($tablename, [$fieldname => $oldvalue]); + + // Loop through each record and update the value. + foreach ($records as $record) { + $record->$fieldname = $newvalue; + $DB->update_record($tablename, $record); + } + + // Update the version number to the latest. + upgrade_plugin_savepoint(true, 2024022701, 'filter', 'autotranslate'); + } + + // Update old source_missing field values. + if ($oldversion < 2024022702) { + // Define the table to be modified. + $table = new xmldb_table('filter_autotranslate_jobs'); + + // Add new fields to the table. + $field1 = new xmldb_field('created_at', XMLDB_TYPE_INTEGER, '11', null, XMLDB_NOTNULL, null, time()); + $field2 = new xmldb_field('modified_at', XMLDB_TYPE_INTEGER, '11', null, XMLDB_NOTNULL, null, time()); + + // Add the fields. + $dbman->add_field($table, $field1); + $dbman->add_field($table, $field2); + + // Set created_at field. + $DB->set_field( + 'filter_autotranslate_jobs', + 'created_at', + time(), + [ "created_at" => null ] + ); + + // Set modified_at field. + $DB->set_field( + 'filter_autotranslate_jobs', + 'modified_at', + time(), + [ "modified_at" => null ] + ); + + // Update the version number to the latest. + upgrade_plugin_savepoint(true, 2024022702, 'filter', 'autotranslate'); + } return true; } diff --git a/filter.php b/filter.php index 9beea16..c6da398 100644 --- a/filter.php +++ b/filter.php @@ -155,13 +155,15 @@ public function filter($text, array $options = []): string { return $text; } - // Check editing mode. - $editing = $PAGE->user_is_editing(); - // Language settings. $sitelang = get_config('core', 'lang'); $currentlang = current_language(); + // Clean the mlang text. + if (str_contains($text, "mlang other")) { + $text = str_replace("mlang other", "mlang $sitelang", $text); + } + // Parsed text // if there is parsed text for the current language // set the text to the parsed text so mlang doesn't get @@ -171,31 +173,46 @@ public function filter($text, array $options = []): string { $langsresult = $this->mlangparser2($text); } - // If mlang is detected, parse it out. - if (!empty($langsresult)) { - // Other mlang was found. - if (array_key_exists('other', $langsresult) && $currentlang === $sitelang) { - $text = $langsresult['other']; - } else if (array_key_exists($currentlang, $langsresult)) { // Current language was found. - $text = $langsresult[$currentlang]; - } - } else if (!array_key_exists($sitelang, $langsresult) && $currentlang === $sitelang) { - // No mlang found and current lang is site lang. + // Check for translated string from core. + // This is to catch translated text that is not coming from multilang. Its possible + // that other components will need to be called and merged in. + $strings = get_string_manager()->load_component_strings('core', $currentlang); + $textistranslation = in_array($text, array_values($strings)); + + // Update the langsresult with appropriate values. + if ($textistranslation) { + // Text is from core. $langsresult[$currentlang] = $text; } else { - // No mlang found and current lang is not site lang - // setting the value as null will notify jobs code below to create a job. + // Text is not from core. + $langsresult[$sitelang] = $text; $langsresult[$currentlang] = null; } + // return empty($langsresult[$currentlang]) ? $langsresult[$sitelang] : $langsresult[$currentlang]; + + // If mlang is detected, parse it out. + // if (!empty($langsresult)) { + // Other mlang was found. + // if (array_key_exists('other', $langsresult) && $currentlang === $sitelang) { + // $text = $langsresult['other']; + // } else if (array_key_exists($currentlang, $langsresult)) { // Current language was found. + // $text = $langsresult[$currentlang]; + // } + // } else if (!array_key_exists($sitelang, $langsresult) && $currentlang === $sitelang) { + // No mlang found and current lang is site lang. + // $langsresult[$currentlang] = $text; + // } else { + // No mlang found and current lang is not site lang + // setting the value as null will notify jobs code below to create a job. + // $langsresult[$currentlang] = null; + // } + // Generate the md5 hash of the current text. $hash = md5($text); // Iterate through each lang results. foreach ($langsresult as $lang => $content) { - // Adjust for other when found. - $lang = $lang === 'other' ? $sitelang : $lang; - // Null content has been found, // kick off job processing. if (!$content && $currentlang === $lang) { @@ -210,7 +227,7 @@ public function filter($text, array $options = []): string { 'hash' => $hash, 'lang' => $lang, 'fetched' => 0, - 'source_missing' => 0, + 'source' => 0, ] ); } @@ -281,12 +298,14 @@ public function filter($text, array $options = []): string { private function mlangparser2($text) { $result = []; $pattern = '/{\s*mlang\s+([^}]+)\s*}(.*?){\s*mlang\s*(?:[^}]+)?}/is'; + $sitelang = get_config('core', 'lang'); if (preg_match_all($pattern, $text, $matches, PREG_SET_ORDER)) { foreach ($matches as $match) { $lang = $match[1]; + $lang = $lang === 'other' ? $sitelang : $lang; $content = $match[2]; - $result[$lang] = $content; + $result[strtolower($lang)] = $content; } } @@ -314,7 +333,7 @@ private function mlangparser($text) { if (preg_match_all('/[a-zA-Z0-9_-]+/', $langblock[1], $langs)) { foreach ($langs[0] as $lang) { $lang = str_replace('-', '_', strtolower($lang)); // Normalize languages. - $langlist[$lang] = trim($langblock[2]); + $langlist[strtolower($lang)] = trim($langblock[2]); } } $result += $langlist; diff --git a/lang/en/filter_autotranslate.php b/lang/en/filter_autotranslate.php index 57be22e..e51a0cf 100644 --- a/lang/en/filter_autotranslate.php +++ b/lang/en/filter_autotranslate.php @@ -79,6 +79,7 @@ $string['fetchtask'] = 'Autotranslation Fetch'; $string['missingapikey'] = 'You have not entered a DeepL API key'; $string['synctask'] = 'Sync DeepL Glossaries'; +$string['checksourcetask'] = "Check source records"; // Capabilities. $string['translate'] = 'Manage autotranslations'; diff --git a/vendor/composer/installed.php b/vendor/composer/installed.php index 4c389a6..7c74b97 100644 --- a/vendor/composer/installed.php +++ b/vendor/composer/installed.php @@ -3,7 +3,7 @@ 'name' => 'kalebheitzman/autotranslate', 'pretty_version' => 'dev-develop', 'version' => 'dev-develop', - 'reference' => '0e2d8aca1501b824a02929d15c0d355b1aceb60e', + 'reference' => 'd3bbe2843498dd2ca9ce52e4085a38aff6c664e7', 'type' => 'library', 'install_path' => __DIR__ . '/../../', 'aliases' => array(), @@ -22,7 +22,7 @@ 'kalebheitzman/autotranslate' => array( 'pretty_version' => 'dev-develop', 'version' => 'dev-develop', - 'reference' => '0e2d8aca1501b824a02929d15c0d355b1aceb60e', + 'reference' => 'd3bbe2843498dd2ca9ce52e4085a38aff6c664e7', 'type' => 'library', 'install_path' => __DIR__ . '/../../', 'aliases' => array(), diff --git a/version.php b/version.php index 414507a..1ca0b06 100644 --- a/version.php +++ b/version.php @@ -24,8 +24,8 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2024022000; // The current plugin version (Date: YYYYMMDDXX). +$plugin->version = 2024022702; // The current plugin version (Date: YYYYMMDDXX). $plugin->requires = 2023042400; // Requires this Moodle version. $plugin->component = 'filter_autotranslate'; // Full name of the plugin (used for diagnostics). $plugin->maturity = MATURITY_ALPHA; -$plugin->release = '4.2.0 (Build: 2024022000)'; +$plugin->release = '4.3.0 (Build: 2024022702)'; From 8c24aed3471f2ce3c23d251f95f397481ac51560 Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Tue, 27 Feb 2024 18:56:28 -0500 Subject: [PATCH 15/18] jobs bug found, potentially... --- filter.php | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/filter.php b/filter.php index c6da398..0d39e17 100644 --- a/filter.php +++ b/filter.php @@ -181,37 +181,26 @@ public function filter($text, array $options = []): string { // Update the langsresult with appropriate values. if ($textistranslation) { - // Text is from core. - $langsresult[$currentlang] = $text; - } else { - // Text is not from core. - $langsresult[$sitelang] = $text; + // Text is from core so we can return it + // without further processing. + return $text; + } + + // Current language for string not detected in multilang. + if (!array_key_exists($currentlang, $langsresult)) { $langsresult[$currentlang] = null; } - // return empty($langsresult[$currentlang]) ? $langsresult[$sitelang] : $langsresult[$currentlang]; - - // If mlang is detected, parse it out. - // if (!empty($langsresult)) { - // Other mlang was found. - // if (array_key_exists('other', $langsresult) && $currentlang === $sitelang) { - // $text = $langsresult['other']; - // } else if (array_key_exists($currentlang, $langsresult)) { // Current language was found. - // $text = $langsresult[$currentlang]; - // } - // } else if (!array_key_exists($sitelang, $langsresult) && $currentlang === $sitelang) { - // No mlang found and current lang is site lang. - // $langsresult[$currentlang] = $text; - // } else { - // No mlang found and current lang is not site lang - // setting the value as null will notify jobs code below to create a job. - // $langsresult[$currentlang] = null; - // } + // Default language for site not detected in multilang. + if (!array_key_exists($sitelang, $langsresult)) { + $langsresult[$sitelang] = $text; + } // Generate the md5 hash of the current text. $hash = md5($text); // Iterate through each lang results. + // Remember: this includes the default site language. foreach ($langsresult as $lang => $content) { // Null content has been found, // kick off job processing. @@ -280,10 +269,11 @@ public function filter($text, array $options = []): string { ); } else if ($record && $currentlang === $lang) { // Translation found, return the text. - $text = $record->text; + return $record->text; } } + $text = empty($langsresult[$currentlang]) ? $langsresult[$sitelang] : $langsresult[$currentlang]; return $text; } From 3af4daab403c4ffb0c51c2d22d294a572366dbbe Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Wed, 28 Feb 2024 15:31:08 -0500 Subject: [PATCH 16/18] refactored filter for bugs --- README.md | 30 ++++++-- classes/task/autotranslate_task.php | 1 + filter.php | 107 +++++++++++++++++++++++++--- 3 files changed, 120 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3f7ecac..c256f6c 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,12 @@ - PHP 8.2+ - Moodle 4.2+ +## Limitations + +- Only supports 2 letter locale codes. +- Limited multilang ({mlang}) support. This filter will process your existing translations in mlang. See the Multilang Support section for more information. +- This filter only supports languages supported in DeepL. + ## Installation - Unzip the plugin in the moodle .../filter/ directory. @@ -15,16 +21,26 @@ This plugin is in an Alpha state and it is highly recommended that you backup yo ## Non-destructive translation -This autotranslation filter does not alter the original text of any of your content on Moodle. It stores all source text and translations in a translation table. The filter works by retrieving the source text from the translation table after it has been saved for the first time. If in the event something goes arwy, disable the filter and all of your original text will be restored immediately. +This autotranslation filter does not alter the original text of any of your content on Moodle. It stores all source and target text in a translation table. The filter works by retrieving the source text from the translation table after it has been saved for the first time. If in the event something goes arwy, disable the filter and all of your original text will be restored immediately. You need to consider your database performance and size when using this plugin. It will effectively double the size of your database if every page of your Moodle site is visited because all source text is being saved as a reference. -## Multlang support +## Multilang support + +This plugin provides limited support for [multilang](https://docs.moodle.org/403/en/Multi-language_content_filter) and [multilang2.](https://moodle.org/plugins/filter_multilang2) A custom parser has been written to find existing translations on your website and store those in the autotranslate table instead of fetching new translations from DeepL. You should enable the translator on each context where you have used multilang in the Autotranslate filter settings. + +### Database prepwork + +If you choose to leave the Multilang and Multilang 2 filters on, you need to do some database prepwork before enabling the Autotranslate filter. You should have a default site language set and know the 2 letter local code for that language. We will use en (English) as an example. + +Navigate to `/admin/tool/replace` and perform a search and replace across your entire database for any `{mlang en}` tags and replace them with `{mlang other}`. It goes without saying that you should backup your database before performing this step. This can also be done via the [CLI.](https://docs.moodle.org/403/en/Search_and_replace_tool) This step is needed for the Autotranslate filter to work correctly. The `other` lang code is needed so that text can be referenced for pages without a translation. + +### Multilang 2 formatting requirements -This plugin provides limited support for [multilang](https://docs.moodle.org/403/en/Multi-language_content_filter) and [multilang2.](https://moodle.org/plugins/filter_multilang2) A custom parser has been written to find existing translations on your website and store those in the autotranslate table instead of fetching new translations from DeepL. This requires that you have only used a single {mlang} tag per translation in your content using the following format: +This filter was developed against a production database where mlang had been wildly used. It is impossible to account for every variation of how someone might have used `{mlang}` tags and whether they used them properly. This filter requires that you have only used a single {mlang} tag per translation in your content using the following format: ``` -{mlang en}Hello{mlang} +{mlang other}Hello{mlang} {mlang es}Hola{mlang} {mlang fr}Bonjour{mlang} ``` @@ -33,13 +49,13 @@ This filter does not support the following structure: ``` Moodle -{mlang en}Hello{mlang} +{mlang other}Hello{mlang} {mlang es}Hola{mlang} {mlang fr}Bonjour{mlang} -{mlang en}Goodbye{mlang} +{mlang other}Goodbye{mlang} ``` -The word Moodle would be stripped out from your translation and you would end up with the words Hola, Bonjour, Goodbye in the translation table. The first Hello would be lost because of the duplicate "en" key. +The word Moodle would be stripped out from your translation and you would end up with the words Hola, Bonjour, Goodbye in the translation table. The first Hello would be lost because of the duplicate "other" key. ## DeepL integration and plugin settings diff --git a/classes/task/autotranslate_task.php b/classes/task/autotranslate_task.php index 335ee40..fe8bf5e 100644 --- a/classes/task/autotranslate_task.php +++ b/classes/task/autotranslate_task.php @@ -101,6 +101,7 @@ public function execute() { // Get the translation. $options = []; $options['formality'] = 'prefer_more'; + $options['tag_handling'] = 'html'; if ($glossaryid) { $options['glossary'] = $glossaryid; } diff --git a/filter.php b/filter.php index 0d39e17..2457269 100644 --- a/filter.php +++ b/filter.php @@ -170,29 +170,51 @@ public function filter($text, array $options = []): string { // stored in the db. $langsresult = $this->mlangparser($text); if (!$langsresult) { + // Look for multilang2 text. $langsresult = $this->mlangparser2($text); } // Check for translated string from core. // This is to catch translated text that is not coming from multilang. Its possible // that other components will need to be called and merged in. - $strings = get_string_manager()->load_component_strings('core', $currentlang); + $component = $this->get_component_name(); + $strings = get_string_manager()->load_component_strings($component, $currentlang); $textistranslation = in_array($text, array_values($strings)); - // Update the langsresult with appropriate values. + // Text is from stringidentifier so we + // return it without further processing. if ($textistranslation) { - // Text is from core so we can return it - // without further processing. return $text; } + // Clean the langresults. + foreach ($langsresult as $lang => $value) { + // Catch bad multilang usage and clean it up. + // Note: there might be value in printing this to the page + // so it can be cleaned up in the editor. + if (str_contains($langsresult[$lang], "{mlang")) { + unset($langsresult[$lang]); + } + + // Empty lang or value detected, unset the array item. + if (empty($lang) || empty($value)) { + unset($langsresult[$lang]); + } + + // Lang codes longer than 2 characters are unsupported. + if (strlen($lang) > 2) { + unset($langsresult[$lang]); + } + } + // Current language for string not detected in multilang. - if (!array_key_exists($currentlang, $langsresult)) { + if (!array_key_exists($currentlang, $langsresult) && $currentlang !== $sitelang) { $langsresult[$currentlang] = null; } // Default language for site not detected in multilang. - if (!array_key_exists($sitelang, $langsresult)) { + // Note: we don't want to enter text based on string identifiers in another language. + if (!array_key_exists($sitelang, $langsresult) && $currentlang === $sitelang) { $langsresult[$sitelang] = $text; } @@ -204,7 +226,7 @@ public function filter($text, array $options = []): string { foreach ($langsresult as $lang => $content) { // Null content has been found, // kick off job processing. - if (!$content && $currentlang === $lang) { + if (!$content && $currentlang === $lang && $currentlang !== $sitelang) { // Check if job exists. $job = $DB->get_record('filter_autotranslate_jobs', ['hash' => $hash, 'lang' => $lang], 'id'); @@ -237,7 +259,7 @@ public function filter($text, array $options = []): string { // Insert the context id record if it does not exist // still create the record even if content is null because // an autotranslation job has been added for the content. - if (!$ctxrecord) { + if (!$ctxrecord && $lang) { $DB->insert_record( 'filter_autotranslate_ctx', [ @@ -255,7 +277,8 @@ public function filter($text, array $options = []): string { // Create a record for the text // do not add null content here. - if (!$record && $content) { + // Note: We don't want to insert stringidentifier content here. + if (!$record && $content && $lang) { $DB->insert_record( 'filter_autotranslate', [ @@ -273,10 +296,72 @@ public function filter($text, array $options = []): string { } } - $text = empty($langsresult[$currentlang]) ? $langsresult[$sitelang] : $langsresult[$currentlang]; + // Never gonna give you up. + if (!empty($langsresult[$currentlang])) { + return $langsresult[$currentlang]; + } + + // Never gonna let you down. + if (!empty($langsresult[$sitelang])) { + return $langsresult[$sitelang]; + } + + // Never gonna run around and desert you. return $text; } + /** + * Get Component Name from the Context + * + * This is used to check against string tranlsations + * in Moodle so that we dont store unnecessary db info. + * + * @return string $component Component Name + */ + private function get_component_name() { + global $DB; + + // Initialize an array to store components. + $components = []; + + // Get the context. + $context = $this->context; + + // Check the type of context and infer associated components. + if ($context instanceof context_system) { + // For the system context, add the system component. + $components[] = 'core_system'; + } else if ($context instanceof context_course) { + // For a course context, add sub-components related to course management. + $components[] = 'core_course'; + } else if ($context instanceof context_module) { + // For a module context, add sub-components related to the specific module. + $module = $DB->get_record('course_modules', ['id' => $context->instanceid]); + if ($module && !empty($module->module)) { + $components[] = 'mod_' . $module->module; + } + } else if ($context instanceof context_block) { + // For a block context, add sub-components related to the specific block. + $blockconfig = get_config('block', $context->instanceid); + if ($blockconfig && !empty($blockconfig->name)) { + $components[] = 'block_' . $blockconfig->name; + } else { + $components[] = 'block_system'; // Fallback block component. + } + } else if ($context instanceof context_coursecat) { + // For a course category context, add sub-components related to course categories. + $components[] = 'core_coursecat'; + } else if ($context instanceof context_user) { + // For a user context, add the user component. + $components[] = 'core_user'; + } + + // Convert the array of components into a string. + $componentstring = implode(',', $components); + + return !empty($componentstring) ? $componentstring : null; + } + /** * Parse {mlang} tags in a given text. * @@ -295,7 +380,7 @@ private function mlangparser2($text) { $lang = $match[1]; $lang = $lang === 'other' ? $sitelang : $lang; $content = $match[2]; - $result[strtolower($lang)] = $content; + $result[strtolower($lang)] = trim($content); } } From b2c0213c41cb6d19c33d76a580d69ab9b5006ebb Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Wed, 28 Feb 2024 15:36:09 -0500 Subject: [PATCH 17/18] fixed double return --- classes/task/check_source_task.php | 1 - 1 file changed, 1 deletion(-) diff --git a/classes/task/check_source_task.php b/classes/task/check_source_task.php index a376b74..d79faa8 100644 --- a/classes/task/check_source_task.php +++ b/classes/task/check_source_task.php @@ -51,7 +51,6 @@ public function execute() { // Your task logic goes here. mtrace("Executing autotranslation check source tasks..."); - // Get 100 existing jobs. $jobs = $DB->get_records('filter_autotranslate_jobs', [ 'fetched' => '0', From 38dffba47790aea9569e9a268edadd661a087ffe Mon Sep 17 00:00:00 2001 From: Kaleb Heitzman Date: Wed, 28 Feb 2024 16:54:38 -0500 Subject: [PATCH 18/18] updated changelog --- CHANGES.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 54d25d0..ba5990b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ # Changelog +## 2024022202 + +- Major Refactor +- Better mlang support +- Fixed various bugs after testing with production data + ## 2024021700 -- Initial Release +- Initial Release