fix: build locale types fully

.eslintrc.js

@@ -153,5 +153,56 @@ module.exports = {
       files: ['ts/node/**/*.ts', 'ts/test/**/*.ts'],
       rules: { 'no-console': 'off', 'import/no-extraneous-dependencies': 'off' },
     },
+    {
+      files: ['ts/localization/*.ts', 'ts/localization/**/*.ts'], // anything in ts/localization has to only reference the files in that folder (this makes it reusable)
+      rules: {
+        'no-restricted-imports': [
+          'error',
+          {
+            patterns: [
+              'a*',
+              'b*',
+              'c*',
+              'd*',
+              'e*',
+              'f*',
+              'g*',
+              'h*',
+              'i*',
+              'j*',
+              'k*',
+              'l*',
+              'm*',
+              'n*',
+              'o*',
+              'p*',
+              'q*',
+              'r*',
+              's*',
+              't*',
+              'u*',
+              'v*',
+              'w*',
+              'x*',
+              'y*',
+              'z*',
+              '0*',
+              '1*',
+              '2*',
+              '3*',
+              '4*',
+              '5*',
+              '6*',
+              '7*',
+              '8*',
+              '9*',
+              '!./*',
+            ], // Disallow everything except ts/localization, this is the worst,
+            // but regexes are broken on our eslint8, and upgrading it means
+            // we need to bump node, which needs to bump electron.... and having '*' makes the other rules droped..
+          },
+        ],
+      },
+    },
   ],
 };

_locales/th/messages.json

@@ -719,7 +719,6 @@
   "searchContacts": "ค้นหาผู้ติดต่อ",
   "searchConversation": "ค้นหาอะไรในการสนทนา",
   "searchEnter": "เขียนที่ค้นหา",
-  "searchMatches": "{count, plural, other [{found_count} จาก # รายการ]}",
   "searchMatchesNone": "ไม่พบข้อมูลเลย",
   "searchMatchesNoneSpecific": "ไม่พบข้อมูลเกี่ยวกับ '{query}",
   "searchMembers": "ค้นหาสมาชิก",

about_preload.js

@@ -7,12 +7,11 @@ const os = require('os');
 const { setupI18n } = require('./ts/util/i18n/i18n');
 
 const config = url.parse(window.location.toString(), true).query;
-const { dictionary, crowdinLocale } = ipcRenderer.sendSync('locale-data');
+const { crowdinLocale } = ipcRenderer.sendSync('locale-data');
 
 window.theme = config.theme;
 window.i18n = setupI18n({
   crowdinLocale,
-  translationDictionary: dictionary,
 });
 
 window.getOSRelease = () =>

actions/setup_and_build/action.yml

@@ -15,7 +15,11 @@ runs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
+
+    - name: Install setuptools for python 3.12
+      shell: bash
+      run: python -m pip install --upgrade pip setuptools
 
     # Not having this will break the windows build because the PATH won't be set by msbuild.
     - name: Add msbuild to PATH

package.json

@@ -36,8 +36,8 @@
     "start-prod": "cross-env NODE_ENV=production NODE_APP_INSTANCE=devprod$MULTI electron .",
     "start-prod:pretty": "cross-env NODE_ENV=production NODE_APP_INSTANCE=devprod$MULTI electron . | npx bunyan",
     "start-dev": "cross-env NODE_ENV=development NODE_APP_INSTANCE=devprod$MULTI electron .",
-    "build-everything": "yarn clean && yarn protobuf && yarn update-git-info && yarn sass && yarn build:locales-soft && tsc && yarn build:workers",
-    "build-everything:soft": "yarn clean && yarn protobuf && yarn update-git-info && yarn sass && yarn build:locales-soft && tsc && yarn build:workers",
+    "build-everything": "python3 --version && yarn clean && yarn protobuf && yarn update-git-info && yarn sass && yarn build:locales-soft && tsc && yarn build:workers",
+    "build-everything:soft": "python3 --version && yarn clean && yarn protobuf && yarn update-git-info && yarn sass && yarn build:locales-soft && tsc && yarn build:workers",
     "build-everything:watch": "yarn clean && yarn protobuf && yarn update-git-info && yarn sass && yarn build:locales-soft && yarn build:workers && yarn tsc -w",
     "start-dev:pretty": "cross-env NODE_ENV=production NODE_APP_INSTANCE=devprod$MULTI electron . | npx bunyan",
     "build:workers": "yarn worker:utils && yarn worker:libsession",

password_preload.js

@@ -6,15 +6,14 @@ const url = require('url');
 const { setupI18n } = require('./ts/util/i18n/i18n');
 
 const config = url.parse(window.location.toString(), true).query;
-const { dictionary, crowdinLocale } = ipcRenderer.sendSync('locale-data');
+const { crowdinLocale } = ipcRenderer.sendSync('locale-data');
 
 // If the app is locked we can't access the database to check the theme.
 window.theme = 'classic-dark';
 window.primaryColor = 'green';
 
 window.i18n = setupI18n({
   crowdinLocale,
-  translationDictionary: dictionary,
 });
 
 window.getEnvironment = () => config.environment;

preload.js

@@ -11,12 +11,12 @@ const _ = require('lodash');
 
 const { setupI18n } = require('./ts/util/i18n/i18n');
 
-const { dictionary, crowdinLocale } = ipc.sendSync('locale-data');
+const { crowdinLocale } = ipc.sendSync('locale-data');
 
 const config = url.parse(window.location.toString(), true).query;
 const configAny = config;
 
-window.i18n = setupI18n({ crowdinLocale, translationDictionary: dictionary });
+window.i18n = setupI18n({ crowdinLocale });
 
 let title = config.name;
 if (config.environment !== 'production') {

tools/localization/generateLocales.py

@@ -17,7 +17,7 @@
   prettyPrintIssuesTable,
   identifyAndPrintOldDynamicVariables, extractFormattingTags,
 )
-from localization.localeTypes import generateLocalesType
+from localization.localeTypes import generateLocalesType, generateLocalesMergedType
 from util.logger import console
 from util.fileUtils import createMappedJsonFileDictionary, writeFile
 
@@ -116,7 +116,7 @@
 
 # Generate the locales type and write it to a file
 if GENERATE_TYPES:
-  generateTypesOutputMessage = generateLocalesType(locales["en"])
+  generateTypesOutputMessage = generateLocalesMergedType(locales)
   console.info(generateTypesOutputMessage)
 
 localeVariables = dict()

tools/localization/localeTypes.py

@@ -1,5 +1,6 @@
 #!/bin/python3
 import re
+from typing import List, Tuple
 
 OUTPUT_FILE = "./ts/localization/locales.ts"
 
@@ -48,6 +49,126 @@ def generate_js_object(data):
     js_object += "}"
     return js_object
 
+def escape_new_lines(value):
+    """
+    Escapes new lines, from "\n" to "\\n".
+    """
+    return value.replace("\n", "\\n")
+
+
+def extract_vars(text):
+    # Use a regular expression to find all strings inside curly braces
+    vars = re.findall(r'\{(.*?)\}', text)
+    return vars
+
+
+def vars_to_record(vars):
+    arr = []
+    for var in vars:
+        to_append = '' + var + ': ' + ('"number"' if var == 'count' or var == 'found_count' else '"string"')
+        if to_append not in arr:
+          arr.append(to_append)
+
+    # print(arr)
+    if not arr:
+        return ''
+    return "{" + ', '.join(arr) + "}"
+
+
+def replace_static_strings(str):
+    # todo make those come from the glossary
+    replaced =  str.replace("{app_name}", "Session")\
+      .replace("{session_download_url}", "https://getsession.org/download")\
+      .replace("{session_download_url}", "GIF")\
+      .replace("{oxen_foundation}", "Oxen Foundation")\
+      .replace("\"", "\\\"")
+    return replaced
+
+
+def args_to_type(args):
+   return args if args else 'undefined,'
+
+
+def generate_type_object(locales):
+    """
+    Generate a JavaScript type from a dictionary.
+
+    Args:
+      data (dict): The dictionary containing key-value pairs.
+
+    Returns:
+      str: A string representation of the JavaScript object.
+    """
+    js_object = "{\n"
+    js_plural_object_container = "{\n"
+    plural_pattern = r"(zero|one|two|few|many|other)\s*\[([^\]]+)\]"
+
+    for key, value_en in locales['en'].items():
+        if value_en.startswith("{count, plural, "):
+            extracted_vars_en = extract_vars(replaced_en)
+            plurals_other = [[locale, replace_static_strings(data.get(key, ""))] for locale, data in locales.items()]
+            en_plurals_with_token = re.findall(plural_pattern, value_en.replace('#', '{count}'))
+
+            if not en_plurals_with_token:
+               raise ValueError("invalid plural string")
+
+            all_locales_plurals = []
+
+            extracted_vars = extract_vars(replace_static_strings(en_plurals_with_token[0][1]))
+            if('count' not in extracted_vars):
+                extracted_vars.append('count')
+
+            for plural in plurals_other:
+              js_plural_object = ""
+
+              locale_key = plural[0].replace("_","-") # 'lo', 'th', 'zh-CN', ....
+              plural_str = plural[1].replace('#', '{count}')
+
+              plurals_with_token = re.findall(plural_pattern, plural_str)
+
+
+              all_locales_strings = []
+              as_record_type_en = vars_to_record(extracted_vars)
+
+              for token, localized_string in plurals_with_token:
+                if localized_string:
+                  to_append = ""
+                  to_append += token
+                  to_append += f": \"{escape_new_lines(localized_string)}\""
+                  all_locales_strings.append(to_append)
+
+              # if that locale doesn't have translation in plurals, add the english hones
+              if not len(all_locales_strings):
+                 for plural_en_token, plural_en_str in en_plurals_with_token:
+                    all_locales_strings.append(f"{plural_en_token}: \"{escape_new_lines(plural_en_str)}\"")
+              js_plural_object += f"    {wrapValue(locale_key)}:"
+              js_plural_object += "{\n      "
+              js_plural_object += ",\n      ".join(all_locales_strings)
+              js_plural_object += "\n    },"
+
+              all_locales_plurals.append(js_plural_object)
+            js_plural_object_container += f'  {wrapValue(key)}: {{\n{"\n".join(all_locales_plurals)}\n    args: {args_to_type(as_record_type_en)}\n  }},\n'
+
+        else:
+          replaced_en = replace_static_strings(value_en)
+          extracted_vars_en = extract_vars(replaced_en)
+          as_record_type_en = vars_to_record(extracted_vars_en)
+          other_locales_replaced_values = [[locale, replace_static_strings(data.get(key, ""))] for locale, data in locales.items()]
+
+          all_locales_strings = []
+          for locale, replaced_val in other_locales_replaced_values:
+            if replaced_val:
+              all_locales_strings.append(f'{wrapValue(locale.replace("_","-"))}: "{escape_new_lines(replaced_val)}"')
+            else:
+              all_locales_strings.append(f'{wrapValue(locale.replace("_","-"))}: "{escape_new_lines(replaced_en)}"')
+
+          # print('key',key, " other_locales_replaced_values:", other_locales_replaced_values)
+          js_object += f'  {wrapValue(key)}: {{\n      {",\n      ".join(all_locales_strings)},\n      args: {args_to_type(as_record_type_en)}\n  }},\n'
+
+    js_object += "}"
+    js_plural_object_container += "}"
+    return js_object,js_plural_object_container
+
 
 DISCLAIMER = """
 // This file was generated by a script. Do not modify this file manually.
@@ -56,17 +177,54 @@ def generate_js_object(data):
 """
 
 
-def generateLocalesType(locale):
+def generateLocalesType(locale, data):
+    """
+    Generate the locales type and write it to a file.
+
+    Args:
+      locale: The locale dictionary containing the localization data.
+    """
+    # write the locale_dict to a file
+    with open(OUTPUT_FILE, "w", encoding='utf-8') as ts_file:
+        ts_file.write(
+            f"{DISCLAIMER}"
+        )
+        ts_file.write(
+            f"export const {locale} = {generate_js_object(data)} as const;\n"
+        )
+        ts_file.write(
+            f"\nexport type Dictionary = typeof en;\n"
+        )
+
+
+    return f"Locales generated at: {OUTPUT_FILE}"
+
+
+def generateLocalesMergedType(locales):
     """
     Generate the locales type and write it to a file.
 
     Args:
       locale: The locale dictionary containing the localization data.
     """
+
     # write the locale_dict to a file
     with open(OUTPUT_FILE, "w", encoding='utf-8') as ts_file:
         ts_file.write(
             f"{DISCLAIMER}"
-            f"export const en = {generate_js_object(locale)} as const;\nexport type Dictionary = typeof en;"
         )
+
+        dicts = generate_type_object(locales)
+
+        dictVar = "simpleDictionary"
+        pluralDictVar = "pluralsDictionary"
+
+
+        ts_file.write(f"""
+export const {dictVar} = {dicts[0]} as const;
+
+export const {pluralDictVar} = {dicts[1]} as const;
+""")
+
     return f"Locales generated at: {OUTPUT_FILE}"
+