Fixes for the bicleaner kind & datasets to make it work with GPU work…

…ers (mozilla#119) * Use relative paths for fetch and artifacts directories to avoid incompatibility between docker-worker and generic-worker * Explicitly add the directory that pip installs entry points to into $PATH For whatever reason, the new generic-worker GPU instances don't have this in $PATH already. * Add bicleaner-ai data packs * Use correct bicleaner pack fetches for bicleaner-ai
mozilla-releng · May 15, 2023 · cd169ef · cd169ef
1 parent 222e541
commit cd169ef
Show file tree

Hide file tree

Showing 2 changed files with 242 additions and 12 deletions.
diff --git a/taskcluster/ci/bicleaner/kind.yml b/taskcluster/ci/bicleaner/kind.yml
@@ -37,15 +37,15 @@ task-defaults:
         max-run-time: 3600
         artifacts:
             - name: public/build
-              path: /builds/worker/artifacts
+              path: artifacts
               type: directory
         env:
             SRC: "{src_locale}"
             TRG: "{trg_locale}"
             # It would be preferable to use $MOZ_FETCHES_DIR here, but these don't
             # get interpreted.
-            CUDA_DIR: /builds/worker/fetches/cuda-toolkit
-            CUDNN_DIR: /builds/worker/fetches/cuda-toolkit
+            CUDA_DIR: fetches/cuda-toolkit
+            CUDNN_DIR: fetches/cuda-toolkit
             COMPRESSION_CMD: zstdmt
             ARTIFACT_EXT: zst
 
@@ -74,18 +74,17 @@ task-defaults:
                 pip install $MOZ_FETCHES_DIR/hunspell-0.5.5-cp310-cp310-linux_x86_64.whl &&
                 pip install $MOZ_FETCHES_DIR/kenlm-0.0.0-cp310-cp310-linux_x86_64.whl &&
                 pip install -r {bicleaner_reqs} &&
+                export PATH=$PATH:~/.local/bin &&
                 $VCS_PATH/pipeline/bicleaner/bicleaner.sh
                 $MOZ_FETCHES_DIR/{dataset_no_slashes}
-                /builds/worker/artifacts/{dataset_no_slashes}
+                artifacts/{dataset_no_slashes}
                 {bicleaner_threshold}
                 {bicleaner_type}
                 {bicleaner_threads}
                 $MOZ_FETCHES_DIR/{src_locale}-{trg_locale}
     dependencies:
         "{provider}": clean-{provider}-{dataset_no_slashes}-{src_locale}-{trg_locale}
     fetches:
-        fetch:
-            - bicleaner-{src_locale}-{trg_locale}
         toolchain:
             - hunspell
             - kenlm
@@ -109,6 +108,9 @@ tasks:
                 type: bicleaner
                 resources:
                     - pipeline/bicleaner/requirements/bicleaner.txt
+        fetches:
+            fetch:
+                - bicleaner-{src_locale}-{trg_locale}
         run:
             command-context:
                 bicleaner_type: bicleaner
@@ -130,6 +132,9 @@ tasks:
                 type: bicleaner-ai
                 resources:
                     - pipeline/bicleaner/requirements/bicleaner-ai.txt
+        fetches:
+            fetch:
+                - bicleaner-ai-{src_locale}-{trg_locale}
         run:
             command-context:
                 bicleaner_type: bicleaner-ai

diff --git a/taskcluster/ci/fetch/bicleaner.yml b/taskcluster/ci/fetch/bicleaner.yml
@@ -2,14 +2,239 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
-bicleaner-ai-full-en-xx:
-    description: bicleaner-ai full en-xx pack
+bicleaner-ai-en-bg:
+    description: bicleaner lite-en-bg pack
     fetch:
         type: static-url
-        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/full-en-xx.tgz
-        sha256: 6429bf2802224e7ae52493a4aef4c0c78fb2b86837f3ebf8a3613432e6227880
-        size: 952926795
-        artifact-name: bicleaner-full-en-xx.tar.zst
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-bg.tgz
+        sha256: df40ee261dd8bb24b0bd81c7dfa2e1c4d9b1a123fb82dd93518b884d10346c28
+        size: 369188943
+        artifact-name: bicleaner-ai-en-bg.tar.zst
+
+bicleaner-ai-en-cs:
+    description: bicleaner lite-en-cs pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-cs.tgz
+        sha256: 7c9f692e341ed6db27bb4fec58fe59968c0f43bfeaa8e7720cd700fc704142b5
+        size: 429288829
+        artifact-name: bicleaner-ai-en-cs.tar.zst
+
+bicleaner-ai-en-da:
+    description: bicleaner lite-en-da pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-da.tgz
+        sha256: 74fecadd06c57932092bb6abf9d066ab77e6bf83bd4d38c1e24bf7369a661991
+        size: 367562892
+        artifact-name: bicleaner-ai-en-da.tar.zst
+
+bicleaner-ai-en-de:
+    description: bicleaner lite-en-de pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-de.tgz
+        sha256: 1a66edfce8493c5cd4238aac77590103bf8c000a2d1b011d62129c1203ce9016
+        size: 503693049
+        artifact-name: bicleaner-ai-en-de.tar.zst
+
+bicleaner-ai-en-el:
+    description: bicleaner lite-en-el pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-el.tgz
+        sha256: 06e52ae97abc4d7cc92789e0285a629ba52af7daf15f4fcae00927f4ed8d56de
+        size: 446093598
+        artifact-name: bicleaner-ai-en-el.tar.zst
+
+bicleaner-ai-en-es:
+    description: bicleaner lite-en-es pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-es.tgz
+        sha256: 35ff078b0c7a08601793a99478bfbe4032f3b56d6a46f0cf4e47aea41257cb95
+        size: 535261323
+        artifact-name: bicleaner-ai-en-es.tar.zst
+
+bicleaner-ai-en-et:
+    description: bicleaner lite-en-et pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-et.tgz
+        sha256: 7468908c246b761d36e3cc57306f7d342cb694b2b72b597c14c354c07e69cc60
+        size: 399644320
+        artifact-name: bicleaner-ai-en-et.tar.zst
+
+bicleaner-ai-en-fi:
+    description: bicleaner lite-en-fi pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-fi.tgz
+        sha256: b658cf5c5e3c718a0b8565cdc8d8872c7ff79493e4bd015ed682aff98f9511db
+        size: 422105777
+        artifact-name: bicleaner-ai-en-fi.tar.zst
+
+bicleaner-ai-en-fr:
+    description: bicleaner lite-en-fr pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-fr.tgz
+        sha256: 1af3675c1525dc35cf4a9a1ce2300aa3c3e5f53a92ef2da5c327596459ff97c2
+        size: 509180872
+        artifact-name: bicleaner-ai-en-fr.tar.zst
+
+bicleaner-ai-en-ga:
+    description: bicleaner lite-en-ga pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-ga.tgz
+        sha256: aefcd387d75924800d75c96131120040501dad74547f2437f4187da82bc55d0b
+        size: 312291448
+        artifact-name: bicleaner-ai-en-ga.tar.zst
+
+bicleaner-ai-en-hr:
+    description: bicleaner lite-en-hr pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-hr.tgz
+        sha256: 52302b29399e278070a3fca2c7633f35c67aa7e51459dab339b87e213403fea1
+        size: 496642588
+        artifact-name: bicleaner-ai-en-hr.tar.zst
+
+bicleaner-ai-en-hu:
+    description: bicleaner lite-en-hu pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-hu.tgz
+        sha256: 9232986b60323384cf95ae08deeea0c009dfc1f5ace85e69187a901a09e0bb87
+        size: 472285319
+        artifact-name: bicleaner-ai-en-hu.tar.zst
+
+bicleaner-ai-en-is:
+    description: bicleaner lite-en-is pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-is.tgz
+        sha256: e7b2871d0d5f377af548762b48db8b7d2442bfca4d44bcbb1fe0246648d1529b
+        size: 381104893
+        artifact-name: bicleaner-ai-en-is.tar.zst
+
+bicleaner-ai-en-it:
+    description: bicleaner lite-en-it pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-it.tgz
+        sha256: 9fdd0f35a05ca15f8e5bc53df26f2cba3f83986b764b6ad4f9d3a98a94c5cb67
+        size: 480925447
+        artifact-name: bicleaner-ai-en-it.tar.zst
+
+bicleaner-ai-en-lt:
+    description: bicleaner lite-en-lt pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-lt.tgz
+        sha256: 83d802b1fa1904fa04f569efdeaf11228cc92af22ed23d2ab946b12ca387c17c
+        size: 380216519
+        artifact-name: bicleaner-ai-en-lt.tar.zst
+
+bicleaner-ai-en-lv:
+    description: bicleaner lite-en-lv pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-lv.tgz
+        sha256: 710ffe6cae4c97808ba267b75295cd2954f9a4753b447fc6b5be4ef2825ac161
+        size: 481751211
+        artifact-name: bicleaner-ai-en-lv.tar.zst
+
+bicleaner-ai-en-mt:
+    description: bicleaner lite-en-mt pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-mt.tgz
+        sha256: 0d08ae53b906810c5b655f4546f8f95c33e365326bfb6322ce3b01993a28112d
+        size: 310023942
+        artifact-name: bicleaner-ai-en-mt.tar.zst
+
+bicleaner-ai-en-nb:
+    description: bicleaner lite-en-nb pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-nb.tgz
+        sha256: 7ca7bd587d220493a7b9c37d6d463bfbf99134387bfa9ad9d1ac8c3302c2d051
+        size: 419041286
+        artifact-name: bicleaner-ai-en-nb.tar.zst
+
+bicleaner-ai-en-nl:
+    description: bicleaner lite-en-nl pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-nl.tgz
+        sha256: 6fd4da690358f4f9c6fc06a97423f4c3371b41dba575b9e88d3187ec25fbe001
+        size: 427505540
+        artifact-name: bicleaner-ai-en-nl.tar.zst
+
+bicleaner-ai-en-nn:
+    description: bicleaner lite-en-nn pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-nn.tgz
+        sha256: b0dd407ffc257f0f485583ba750897dc8abb3e54927b975ee8967381dfa18d1f
+        size: 422402790
+        artifact-name: bicleaner-ai-en-nn.tar.zst
+
+bicleaner-ai-en-pl:
+    description: bicleaner lite-en-pl pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-pl.tgz
+        sha256: 0eb5cd21fb714af81854eb1ed60b18e99a8f59b9bda495df2dc0ff620665c5f1
+        size: 546487672
+        artifact-name: bicleaner-ai-en-pl.tar.zst
+
+bicleaner-ai-en-pt:
+    description: bicleaner lite-en-pt pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-pt.tgz
+        sha256: b32ac876a12146957e65960a4e0a3da41630f280d6c3ad1e8306c8c0ed853d51
+        size: 502639758
+        artifact-name: bicleaner-ai-en-pt.tar.zst
+
+bicleaner-ai-en-ro:
+    description: bicleaner lite-en-ro pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-ro.tgz
+        sha256: 4ab30b6eab8a00d4324737b32e56f2050ab14c0426ac8e9e753660b4b853d1b9
+        size: 444265818
+        artifact-name: bicleaner-ai-en-ro.tar.zst
+
+bicleaner-ai-en-sk:
+    description: bicleaner lite-en-sk pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-sk.tgz
+        sha256: 22756e536ade06f5b4f7cebf4c6ab964c48a1abd1c81d68560a1d11fb2c0f613
+        size: 499326003
+        artifact-name: bicleaner-ai-en-sk.tar.zst
+
+bicleaner-ai-en-sl:
+    description: bicleaner lite-en-sl pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-sl.tgz
+        sha256: 182ef07d8d5548ea14d27edbe65900502d6a1a8970df9aa38ce0e215907131f3
+        size: 439849151
+        artifact-name: bicleaner-ai-en-sl.tar.zst
+
+bicleaner-ai-en-sv:
+    description: bicleaner lite-en-sv pack
+    fetch:
+        type: static-url
+        url: https://github.com/bitextor/bicleaner-ai-data/releases/download/v2.0/lite-en-sv.tgz
+        sha256: 62418c0768f55e4787ffdeae310695ca658047aefb2aa4e60ba28e75519bfdb3
+        size: 407742801
+        artifact-name: bicleaner-ai-en-sv.tar.zst
 
 bicleaner-en-bg:
     description: bicleaner en-bg pack