Merge pull request #8866 from ethereum/i18n-extract-strings

React i18next - with translation strings extraction

.gitignore

@@ -76,6 +76,8 @@ yarn-error.log
 src/data/contributors.json
 # These files are generated by `yarn merge-translations` command
 src/intl/*.json
+i18n/locales
+i18n/merged
 # Auto generated code when gatsby build the site
 src/gatsby-types.d.ts
 

docs/locales-process.md

@@ -0,0 +1,9 @@
+# Locales generation process
+
+Every time `yarn build` or `yarn start` is executed, the following process is
+going to be triggered as well:
+
+<img src="./locales.png">
+
+With this process, we reduce the amount of text we bundle on each page since we
+are querying only the necessary translations that each page needs.

gatsby-browser.tsx

@@ -4,10 +4,6 @@
  * See: https://www.gatsbyjs.org/docs/browser-apis/
  */
 
-import React from "react"
-import browserLang from "browser-lang"
-import { withPrefix, GatsbyBrowser } from "gatsby"
-
 import Prism from "prism-react-renderer/prism"
 ;(typeof global !== "undefined" ? global : window).Prism = Prism
 
@@ -16,53 +12,6 @@ import "@formatjs/intl-locale/polyfill"
 import "@formatjs/intl-numberformat/polyfill"
 import "@formatjs/intl-numberformat/locale-data/en"
 
-import Layout from "./src/components/Layout"
-import {
-  supportedLanguages,
-  defaultLanguage,
-  isLang,
-} from "./src/utils/languages"
-import { IS_DEV } from "./src/utils/env"
-import { Context } from "./src/types"
-
 // Default languages included:
 // https://github.com/FormidableLabs/prism-react-renderer/blob/master/src/vendor/prism/includeLangs.js
 require("prismjs/components/prism-solidity")
-
-// Prevents <Layout/> from unmounting on page transitions
-// https://www.gatsbyjs.com/docs/layout-components/#how-to-prevent-layout-components-from-unmounting
-// @ts-ignore: returning `null` is not accepted by the `GatsbyBrowser` type def.
-export const wrapPageElement: GatsbyBrowser<
-  any,
-  Context
->["wrapPageElement"] = ({ element, props }) => {
-  const { location, pageContext } = props
-  const { pathname, search } = location
-  const { originalPath } = pageContext
-
-  const [, pathLocale] = pathname.split("/")
-
-  // client side redirect on paths that don't have a locale in them. Most useful
-  // on dev env where we don't have server redirects
-  if (IS_DEV && !isLang(pathLocale)) {
-    let detected =
-      window.localStorage.getItem("eth-org-language") ||
-      browserLang({
-        languages: supportedLanguages,
-        fallback: defaultLanguage,
-      })
-
-    if (!isLang(detected)) {
-      detected = defaultLanguage
-    }
-
-    const queryParams = search || ""
-    const newUrl = withPrefix(`/${detected}${originalPath}${queryParams}`)
-    window.localStorage.setItem("eth-org-language", detected)
-    window.location.replace(newUrl)
-
-    return null
-  }
-
-  return <Layout {...props}>{element}</Layout>
-}

gatsby-config.ts

@@ -35,24 +35,6 @@ const config: GatsbyConfig = {
     editContentUrl: `https://github.com/ethereum/ethereum-org-website/tree/dev/`,
   },
   plugins: [
-    // i18n support
-    {
-      resolve: `gatsby-theme-i18n`,
-      options: {
-        defaultLang: defaultLanguage,
-        prefixDefault: true,
-        locales: supportedLanguages.length
-          ? supportedLanguages.join(" ")
-          : null,
-        configPath: path.resolve(`./i18n/config.json`),
-      },
-    },
-    {
-      resolve: `gatsby-theme-i18n-react-intl`,
-      options: {
-        defaultLocale: `./src/intl/en.json`,
-      },
-    },
     // Web app manifest
     {
       resolve: `gatsby-plugin-manifest`,
@@ -266,6 +248,59 @@ const config: GatsbyConfig = {
         generateMatchPathRewrites: false,
       },
     },
+    // i18n support
+    {
+      resolve: `gatsby-source-filesystem`,
+      options: {
+        path: path.resolve(`./i18n/locales`),
+        name: `locale`,
+      },
+    },
+    // Wraps the entire page with a custom layout component
+    // Note: keep this before the i18n plugin declaration in order to have the
+    // i18n provider wrapping the layout component
+    {
+      resolve: `gatsby-plugin-layout`,
+      options: {
+        component: path.resolve(`./src/components/Layout`),
+      },
+    },
+    {
+      resolve: `gatsby-plugin-react-i18next`,
+      options: {
+        localeJsonSourceName: `locale`, // name given to `gatsby-source-filesystem` plugin.
+        languages: supportedLanguages,
+        defaultLanguage,
+        generateDefaultLanguagePage: true,
+        redirect: false,
+        siteUrl,
+        trailingSlash: "always",
+        // i18next options
+        i18nextOptions: {
+          fallbackLng: defaultLanguage,
+          interpolation: {
+            escapeValue: false,
+          },
+          load: "currentOnly",
+          lowerCaseLng: true,
+          cleanCode: true,
+          react: {
+            transSupportBasicHtmlNodes: true,
+            transKeepBasicHtmlNodesFor: [
+              "br",
+              "strong",
+              "i",
+              "bold",
+              "b",
+              "em",
+              "sup",
+            ],
+          },
+          keySeparator: false,
+          nsSeparator: false,
+        },
+      },
+    },
   ],
   // https://www.gatsbyjs.com/docs/reference/release-notes/v2.28/#feature-flags-in-gatsby-configjs
   flags: {

gatsby-node.ts

@@ -10,7 +10,7 @@ import type { Context } from "./src/types"
 
 import * as Schema from "./src/schema"
 
-import mergeTranslations from "./src/scripts/mergeTranslations"
+import createLocales from "./src/scripts/createLocales"
 import copyContributors from "./src/scripts/copyContributors"
 
 import {
@@ -284,17 +284,23 @@ export const createPages: GatsbyNode<any, Context>["createPages"] = async ({
             component: path.resolve(`src/templates/${template}.tsx`),
             context: {
               language: lang,
+              languagesToFetch: [lang],
               slug: langSlug,
               ignoreTranslationBanner: isLegal,
               isLegal: isLegal,
               isOutdated: false,
               isContentEnglish: true,
               relativePath, // Use English path for template MDX query
-              // gatsby i18n theme context
-              locale: lang,
-              hrefLang: lang,
-              originalPath: langSlug.slice(3),
-              dateFormat: "MM/DD/YYYY",
+              // gatsby i18n plugin
+              i18n: {
+                language: lang,
+                languages: supportedLanguages,
+                defaultLanguage: defaultLanguage,
+                generateDefaultLanguagePage: false,
+                routed: true,
+                originalPath: langSlug.slice(3),
+                path: langSlug,
+              },
             },
           })
         }
@@ -306,15 +312,21 @@ export const createPages: GatsbyNode<any, Context>["createPages"] = async ({
       component: path.resolve(`src/templates/${template}.tsx`),
       context: {
         language,
+        languagesToFetch: [language],
         slug,
         isOutdated: !!node.fields.isOutdated,
         isDefaultLang: language === defaultLanguage,
         relativePath,
-        // gatsby i18n theme context
-        locale: language,
-        hrefLang: language,
-        originalPath: slug.slice(3),
-        dateFormat: "MM/DD/YYYY",
+        // gatsby i18n plugin
+        i18n: {
+          language,
+          languages: supportedLanguages,
+          defaultLanguage,
+          generateDefaultLanguagePage: false,
+          routed: true,
+          originalPath: slug.slice(3),
+          path: slug,
+        },
       },
     })
   })
@@ -344,22 +356,26 @@ export const createPages: GatsbyNode<any, Context>["createPages"] = async ({
           page,
           lang
         )
-
         const slug = `/${lang}${originalPath}`
-
         createPage<Context>({
           path: slug,
           component: path.resolve(`src/pages-conditional/${page}.tsx`),
           context: {
             language: lang,
+            languagesToFetch: [lang],
             slug,
             isContentEnglish,
             isOutdated,
-            // gatsby i18n theme context
-            locale: lang,
-            hrefLang: lang,
-            originalPath,
-            dateFormat: "MM/DD/YYYY",
+            // gatsby i18n plugin
+            i18n: {
+              language: lang,
+              languages: supportedLanguages,
+              defaultLanguage,
+              generateDefaultLanguagePage: false,
+              routed: true,
+              originalPath,
+              path: slug,
+            },
           },
         })
       }
@@ -376,51 +392,68 @@ export const onCreatePage: GatsbyNode<any, Context>["onCreatePage"] = async ({
 }) => {
   const { createPage, deletePage, createRedirect } = actions
 
-  const isDefaultLang = page.path.startsWith(`/${defaultLanguage}`)
-
-  if (isDefaultLang) {
-    const path = page.path.slice(3)
-
-    if (IS_DEV) {
-      // create routes without the lang prefix e.g. `/{path}` as our i18n plugin
-      // only creates `/{lang}/{path}` routes. This is useful on dev env to avoid
-      // getting a 404 since we don't have server side redirects
-      createPage({ ...page, path })
-    }
-
-    if (!IS_DEV && !path.match(/^\/404(\/|.html)$/)) {
-      // on prod, indicate our servers to redirect the root paths to the
-      // `/{defaultLang}/{path}`
-      createRedirect({
-        ...commonRedirectProps,
-        fromPath: path,
-        toPath: page.path,
-      })
-    }
-  }
-
   if (!page.context) {
     return
   }
 
-  const isTranslated = page.context.locale !== defaultLanguage
-  const hasNoContext = page.context.isOutdated === undefined
+  // these are the native Gatsby pages (those living under `/pages`)
+  // which do not pass through the `createPages` hook thus they don't have our
+  // custom context in them
+  const isPageWithoutCustomContext = page.context.isOutdated === undefined
 
-  if (isTranslated && hasNoContext) {
+  if (isPageWithoutCustomContext) {
+    const { language, i18n } = page.context
+    const isDefaultLang = language === defaultLanguage
+
+    // as we don't have our custom context for this page, we calculate & add it
+    // later to them
     const { isOutdated, isContentEnglish } = await checkIsPageOutdated(
-      page.context.originalPath,
-      page.context.locale
+      i18n.originalPath,
+      language
     )
-    deletePage(page)
-    createPage<Context>({
+
+    let newPage = {
       ...page,
       context: {
         ...page.context,
+        languagesToFetch: [language],
         isOutdated,
         //display TranslationBanner for translation-component pages that are still in English
         isContentEnglish,
       },
-    })
+    }
+
+    // there seems to be a bug in the i18n plugin where 404 pages get a
+    // duplicated `/lang` in their `matchPath`s
+    if (newPage.matchPath?.includes(`/${language}/${language}/*`)) {
+      newPage = { ...newPage, matchPath: `/${language}/*` }
+    }
+
+    // on dev, we will have 2 pages for the default lang
+    // - 1 for the ones with the prefix `/{defaultLang}/learn/`
+    // - 1 for the ones without the prefix `/learn/`
+    //   we do this to avoid having a 404 on those without the prefix since in
+    //   dev we don't have the redirects from the server
+    deletePage(page)
+
+    if (IS_DEV) {
+      createPage<Context>(newPage)
+    }
+
+    // `routed` means that the page have the lang prefix on the url
+    // e.g. `/en/learn` or `/en`
+    if (!IS_DEV && i18n.routed) {
+      createPage<Context>(newPage)
+
+      const rootPath = page.path.slice(3)
+      if (isDefaultLang && !rootPath.match(/^\/404(\/|.html)$/)) {
+        createRedirect({
+          ...commonRedirectProps,
+          fromPath: rootPath,
+          toPath: page.path,
+        })
+      }
+    }
   }
 }
 
@@ -446,9 +479,11 @@ export const createSchemaCustomization: GatsbyNode["createSchemaCustomization"]
     createTypes([...Object.keys(sdls).map((sdlKey) => sdls[sdlKey])])
   }
 
-export const onPreBootstrap: GatsbyNode["onPreBootstrap"] = ({ reporter }) => {
-  mergeTranslations()
-  reporter.info(`Merged translations saved`)
+export const onPreBootstrap: GatsbyNode["onPreBootstrap"] = async ({
+  reporter,
+}) => {
+  await createLocales()
+  reporter.info(`Created locales`)
   copyContributors()
   reporter.info(`Contributors copied`)
 }