Merge pull request #921 from openedx/bmtcril/move_translations

build: Move translations pull to Docker build time

Makefile

@@ -85,12 +85,6 @@ release-push:
 	git push origin $(TAG)
 
 ###### Additional commands
-pull_translations: translation-requirements
-	atlas pull -g $(OPENEDX_ATLAS_ARGS) translations/tutor-contrib-aspects/tutoraspects/*:tutoraspects/templates/
-
-	@echo "Translations have been pulled via Atlas."
-	python tutoraspects/translations/translate.py . compile
-
 extract_translations: translation-requirements
 	python tutoraspects/translations/translate.py . extract
 

tutoraspects/templates/aspects/build/aspects-superset/Dockerfile

@@ -8,6 +8,8 @@ FROM apache/superset:4.0.1
 USER root
 
 COPY ./requirements.txt /app/requirements.txt
+COPY ./localization/compile_translations.py /app/localization/compile_translations.py
+COPY ./localization/datasets_strings.yaml /app/localization/datasets_strings.yaml
 
 RUN pip install -r /app/requirements.txt
 
@@ -18,6 +20,7 @@ ARG FIREFOX_VERSION=117.0.1
 
 RUN apt-get update -q \
     && apt-get install -yq --no-install-recommends \
+        git \
         libnss3 \
         libdbus-glib-1-2 \
         libgtk-3-0 \
@@ -33,6 +36,11 @@ RUN apt-get update -q \
     && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/* /var/[log,tmp]/* /tmp/* && apt-get clean
 
 COPY ./openedx-assets /app/openedx-assets
-COPY ./localization/ /app/localization/
+
+# Pull latest aspects translations from openedx-tranlsations repository
+RUN atlas pull translations/tutor-contrib-aspects/tutoraspects/templates/aspects/apps/superset/conf/locale:/app/localization/
+
+# combine the /app/localization/ files into a single `localization/locale.yaml` file in a way Aspects can use
+RUN python /app/localization/compile_translations.py
 
 USER superset

tutoraspects/templates/aspects/build/aspects-superset/localization/compile_translations.py

@@ -0,0 +1,71 @@
+import os
+import glob
+import shutil
+
+import ruamel.yaml
+import ruamel.yaml.comments
+
+yaml = ruamel.yaml.YAML()
+
+
+def recursive_sort_mappings(s):
+    """Given a ruamel yaml object, recursively sort all mappings in order."""
+    if isinstance(s, list):
+        for elem in s:
+            recursive_sort_mappings(elem)
+        return
+    if not isinstance(s, dict):
+        return
+    for key in sorted(s, reverse=True):
+        value = s.pop(key)
+        recursive_sort_mappings(value)
+        s.insert(0, key, value)
+
+def compile_translations():
+    """
+    Combine translated files into the single file we use for translation.
+
+    This should be called after we pull translations using Atlas, see the
+    pull_translations make target.
+    """
+    translations_path = "/app/localization/"
+
+    all_translations = ruamel.yaml.comments.CommentedMap()
+    yaml_files = glob.glob(os.path.join(translations_path, "**/locale.yaml"))
+
+    for file_path in yaml_files:
+        lang = file_path.split(os.sep)[-2]
+        with open(file_path, "r", encoding="utf-8") as asset_file:
+            loc_str = asset_file.read()
+        loaded_strings = yaml.load(loc_str)
+
+        # Sometimes translated files come back with "en" as the top level
+        # key, but still translated correctly.
+        try:
+            all_translations[lang] = loaded_strings[lang]
+        except KeyError:
+            all_translations[lang] = loaded_strings["en"]
+
+        if None in all_translations[lang]:
+            all_translations[lang].pop(None)
+
+    out_path = "/app/localization/locale.yaml"
+
+    print(f"Writing all translations out to {out_path}")
+    with open(out_path, "w", encoding="utf-8") as outfile:
+        outfile.write("---\n")
+        # If we don't use an extremely large width, the jinja in our translations
+        # can be broken by newlines. So we use the largest number there is.
+        recursive_sort_mappings(all_translations)
+        yaml.dump(all_translations, outfile)
+        outfile.write("\n{{ patch('superset-extra-asset-translations')}}\n")
+
+    # We remove these files to avoid confusion about where translations are coming
+    # from, and because otherwise we will need to re-save them with the large
+    # width as above to avoid Jinja parsing errors.
+    print("Removing downloaded translations files... ")
+    # shutil.rmtree(translations_path)
+
+
+if __name__ == "__main__":
+    compile_translations()