Combine split_by_provider and dataset_substitution transforms into a …

…more general from_datasets one (mozilla#118)
mozilla-releng · May 12, 2023 · 222e541 · 222e541
1 parent 3f023cf
commit 222e541
Show file tree

Hide file tree

Showing 6 changed files with 217 additions and 130 deletions.
diff --git a/taskcluster/ci/bicleaner/kind.yml b/taskcluster/ci/bicleaner/kind.yml
@@ -2,14 +2,11 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
-# TODO: this may not be sude for all locale pairs? or not for all dataset types?
-# TODO: this this run on large instances? gpu?
 
 loader: taskgraph.loader.transform:loader
 
 transforms:
-    - translations_taskgraph.transforms.split_by_provider:transforms
-    - translations_taskgraph.transforms.dataset_substitutions:transforms
+    - translations_taskgraph.transforms.from_datasets:transforms
     - translations_taskgraph.transforms.command_context_from_params:transforms
     - taskgraph.transforms.job:transforms
     - translations_taskgraph.transforms.cache:transforms
@@ -28,13 +25,14 @@ task-defaults:
                 - pipeline/bicleaner/bicleaner.sh
             parameters:
                 - bicleaner_threshold
-    substitution-fields:
-        - description
-        - name
-        - dependencies
-        - fetches
-        - treeherder.symbol
-        - worker.env
+    dataset-config:
+        substitution-fields:
+            - description
+            - name
+            - dependencies
+            - fetches
+            - treeherder.symbol
+            - worker.env
     worker:
         max-run-time: 3600
         artifacts:
@@ -123,6 +121,10 @@ tasks:
         worker-type: t-linux-v100-gpu
         treeherder:
             platform: bicleaner-ai/opt
+        dataset-config:
+            exclude-locales:
+                - src: en
+                  trg: ru
         attributes:
             cache:
                 type: bicleaner-ai

diff --git a/taskcluster/ci/clean/kind.yml b/taskcluster/ci/clean/kind.yml
@@ -6,7 +6,7 @@
 loader: taskgraph.loader.transform:loader
 
 transforms:
-    - translations_taskgraph.transforms.dataset_substitutions:transforms
+    - translations_taskgraph.transforms.from_datasets:transforms
     - taskgraph.transforms.job:transforms
     - translations_taskgraph.transforms.cache:transforms
     - taskgraph.transforms.cached_tasks:transforms
@@ -27,13 +27,14 @@ task-defaults:
                 - pipeline/clean/tools/clean_parallel.py
                 - pipeline/clean/tools/langid_fasttext.py
     worker-type: b-linux
-    substitution-fields:
-        - description
-        - name
-        - dependencies
-        - fetches
-        - treeherder.symbol
-        - worker.env
+    dataset-config:
+        substitution-fields:
+            - description
+            - name
+            - dependencies
+            - fetches
+            - treeherder.symbol
+            - worker.env
     worker:
         docker-image: {"in-tree": "train"}
         max-run-time: 3600
@@ -55,6 +56,8 @@ task-defaults:
         platform: clean/opt
     run:
         using: run-task
+        # Include this so `from_datasets` will add a number of other values to it.
+        command-context: {}
         command:
             - bash
             - -c
@@ -70,16 +73,24 @@ task-defaults:
 
 tasks:
     flores-{dataset}-{src_locale}-{trg_locale}:
-        provider: flores
+        dataset-config:
+            include-datasets:
+                flores: {}
 
     sacrebleu-{dataset}-{src_locale}-{trg_locale}:
-        provider: sacrebleu
+        dataset-config:
+            include-datasets:
+                sacrebleu: {}
 
     opus-{dataset_no_slashes}-{src_locale}-{trg_locale}:
-        provider: opus
+        dataset-config:
+            include-datasets:
+                opus: {}
 
     mtdata-{dataset}-{src_locale}-{trg_locale}:
-        provider: mtdata
+        dataset-config:
+            include-datasets:
+                mtdata: {}
         attributes:
             cache:
                 resources:
@@ -96,4 +107,6 @@ tasks:
                     - pipeline/clean/fixes/mtdata_OPUS_UNPC_v1_0.fr.sh
 
     news-crawl-{dataset}-{src_locale}-{trg_locale}:
-        provider: news-crawl
+        dataset-config:
+            include-datasets:
+                news-crawl: {}
diff --git a/taskcluster/ci/dataset/kind.yml b/taskcluster/ci/dataset/kind.yml
@@ -9,7 +9,7 @@
 loader: taskgraph.loader.transform:loader
 
 transforms:
-    - translations_taskgraph.transforms.dataset_substitutions:transforms
+    - translations_taskgraph.transforms.from_datasets:transforms
     - taskgraph.transforms.job:transforms
     - translations_taskgraph.transforms.cache:transforms
     - taskgraph.transforms.cached_tasks:transforms
@@ -20,10 +20,11 @@ task-defaults:
     attributes:
         cache:
             type: dataset
-    substitution-fields:
-        - name
-        - label
-        - treeherder.symbol
+    dataset-config:
+        substitution-fields:
+            - name
+            - label
+            - treeherder.symbol
     worker:
         docker-image: {in-tree: toolchain-build}
         max-run-time: 1800
@@ -41,12 +42,16 @@ task-defaults:
     run-on-tasks-for: []
     run:
         using: run-task
+        # Include this so `from_datasets` will add a number of other values to it.
+        command-context: {}
 
 tasks:
     flores:
         description: Fetch flores101 dataset
         label: dataset-flores-{dataset}-{src_locale}-{trg_locale}
-        provider: flores
+        dataset-config:
+            include-datasets:
+                flores: {}
         attributes:
             cache:
                 resources:
@@ -60,7 +65,9 @@ tasks:
     sacrebleu:
         description: Fetch sacrebleu dataset
         label: dataset-sacrebleu-{dataset}-{src_locale}-{trg_locale}
-        provider: sacrebleu
+        dataset-config:
+            include-datasets:
+                sacrebleu: {}
         attributes:
             cache:
                 resources:
@@ -75,7 +82,9 @@ tasks:
         description: Fetch opus dataset
         # No slashes version of dataset used here because slashes break caches
         label: dataset-opus-{dataset_no_slashes}-{src_locale}-{trg_locale}
-        provider: opus
+        dataset-config:
+            include-datasets:
+                opus: {}
         attributes:
             cache:
                 resources:
@@ -89,7 +98,9 @@ tasks:
     mtdata:
         description: Fetch mtdata dataset
         label: dataset-mtdata-{dataset}-{src_locale}-{trg_locale}
-        provider: mtdata
+        dataset-config:
+            include-datasets:
+                mtdata: {}
         attributes:
             cache:
                 resources:
@@ -103,7 +114,9 @@ tasks:
     news-crawl:
         description: Fetch news-crawl dataset
         label: dataset-news-crawl-{dataset}-{src_locale}-{trg_locale}
-        provider: news-crawl
+        dataset-config:
+            include-datasets:
+                news-crawl: {}
         attributes:
             cache:
                 resources:

diff --git a/taskcluster/translations_taskgraph/transforms/dataset_substitutions.py b/taskcluster/translations_taskgraph/transforms/dataset_substitutions.py