Use ndlaskip as tag to not translate text

package.json

@@ -99,6 +99,7 @@
     "@ndla/video-search": "^8.0.76-alpha.0",
     "@tanstack/react-query": "5.62.3",
     "auth0-js": "^9.22.1",
+    "cheerio": "^1.0.0-rc.12",
     "compression": "^1.7.4",
     "cross-fetch": "^3.1.5",
     "date-fns": "2.30.0",

src/components/FileUploader/FileUploader.tsx

@@ -117,14 +117,18 @@ const FileUploader = ({ onFileSave, close }: Props) => {
                     // Bug in formik's setError function requiring setTimeout to make it work,
                     // as discussed here: https://github.com/jaredpalmer/formik/discussions/3870
                     if (fileErrors.includes("FILE_INVALID_TYPE")) {
-                      const errorMessage = `${t("form.file.fileUpload.genericError")}: ${t("form.file.fileUpload.fileTypeInvalidError")}`;
+                      const errorMessage = `${t("form.file.fileUpload.genericError")}: ${t(
+                        "form.file.fileUpload.fileTypeInvalidError",
+                      )}`;
                       setTimeout(() => {
                         helpers.setError(errorMessage);
                       }, 0);
                       return;
                     }
                     if (fileErrors.includes("TOO_MANY_FILES")) {
-                      const errorMessage = `${t("form.file.fileUpload.genericError")}: ${t("form.file.fileUpload.tooManyError")}`;
+                      const errorMessage = `${t("form.file.fileUpload.genericError")}: ${t(
+                        "form.file.fileUpload.tooManyError",
+                      )}`;
                       setTimeout(() => {
                         helpers.setError(errorMessage);
                       }, 0);

src/components/NynorskTranslateProvider.tsx

@@ -25,8 +25,8 @@ export interface TranslateType {
   type: "text" | "html";
 }
 
-const domParser = new DOMParser();
-const xmlSerializer = new XMLSerializer();
+//const domParser = new DOMParser();
+//const xmlSerializer = new XMLSerializer();
 
 export const NynorskTranslateProvider = ({ children }: Props) => {
   const translateState = useState<boolean>(false);
@@ -64,13 +64,7 @@ export const useTranslateToNN = () => {
         const content = get(element, field);
         if (content) {
           const isArray = Array.isArray(content);
-          // Our backend uses Jsoup to encode html. However, > is not encoded, and nynodata expects it to be. As such, we have to parse
-          // the entire html string and reencode it using an xmlSerializer.
-          const parsed =
-            type === "html" && !isArray
-              ? xmlSerializer.serializeToString(domParser.parseFromString(content, "text/html").body!)
-              : content;
-          acc[field] = { content: parsed, type, isArray };
+          acc[field] = { content, type, isArray };
         }
         return acc;
       }, {});

src/components/SlateEditor/plugins/video/SlateVideo.tsx

@@ -124,7 +124,9 @@ const SlateVideo = ({ attributes, element, editor, children }: Props) => {
               variant="secondary"
               title={t("form.video.brightcove")}
               aria-label={t("form.video.brightcove")}
-              to={`https://studio.brightcove.com/products/videocloud/media/videos/${embed.embedData.videoid.split("&t=")[0]}`}
+              to={`https://studio.brightcove.com/products/videocloud/media/videos/${
+                embed.embedData.videoid.split("&t=")[0]
+              }`}
               size="small"
             >
               <LinkMedium />

src/containers/AudioUploader/components/AudioContent.tsx

@@ -110,7 +110,9 @@ const AudioContent = () => {
                     const fileErrors = details.files?.[0]?.errors;
                     if (!fileErrors) return;
                     if (fileErrors.includes("FILE_INVALID_TYPE")) {
-                      const errorMessage = `${t("form.audio.fileUpload.genericError")}: ${t("form.audio.fileUpload.fileTypeInvalidError")}`;
+                      const errorMessage = `${t("form.audio.fileUpload.genericError")}: ${t(
+                        "form.audio.fileUpload.fileTypeInvalidError",
+                      )}`;
                       // Bug in formik's setError function requiring setTimeout to make it work,
                       // as discussed here: https://github.com/jaredpalmer/formik/discussions/3870
                       setTimeout(() => {

src/containers/ImageUploader/components/ImageContent.tsx

@@ -93,14 +93,18 @@ const ImageContent = ({ language }: Props) => {
                     const fileErrors = details.files?.[0]?.errors;
                     if (!fileErrors) return;
                     if (fileErrors.includes("FILE_TOO_LARGE")) {
-                      const errorMessage = `${t("form.image.fileUpload.genericError")}: ${t("form.image.fileUpload.tooLargeError")}`;
+                      const errorMessage = `${t("form.image.fileUpload.genericError")}: ${t(
+                        "form.image.fileUpload.tooLargeError",
+                      )}`;
                       setTimeout(() => {
                         helpers.setError(errorMessage);
                       }, 0);
                       return;
                     }
                     if (fileErrors.includes("FILE_INVALID_TYPE")) {
-                      const errorMessage = `${t("form.image.fileUpload.genericError")}: ${t("form.image.fileUpload.fileTypeInvalidError")}`;
+                      const errorMessage = `${t("form.image.fileUpload.genericError")}: ${t(
+                        "form.image.fileUpload.fileTypeInvalidError",
+                      )}`;
                       setTimeout(() => {
                         helpers.setError(errorMessage);
                       }, 0);

src/containers/SearchPage/components/results/SearchContent.tsx

@@ -238,9 +238,11 @@ const SearchContent = ({ content, locale, subjects, responsibleName }: Props) =>
                 target="_blank"
                 aria-label={t("form.workflow.published")}
                 title={t("form.workflow.published")}
-                to={`${config.ndlaFrontendDomain}/${content.learningResourceType === "concept" || content.learningResourceType === "gloss" ? "concept" : "article"}/${
-                  content.id
-                }`}
+                to={`${config.ndlaFrontendDomain}/${
+                  content.learningResourceType === "concept" || content.learningResourceType === "gloss"
+                    ? "concept"
+                    : "article"
+                }/${content.id}`}
               >
                 <CheckLine />
               </SafeLinkIconButton>

src/containers/WelcomePage/components/worklist/WorkList.tsx

@@ -160,9 +160,15 @@ const WorkList = ({ ndlaId }: Props) => {
       }}
     >
       <TabsList>
-        <TabsTrigger value="articles">{`${t("taxonomy.resources")} (${searchQuery.data?.totalCount ?? 0})`}</TabsTrigger>
-        <TabsTrigger value="onHold">{`${t("welcomePage.workList.onHold")} (${searchOnHoldQuery.data?.totalCount ?? 0})`}</TabsTrigger>
-        <TabsTrigger value="concepts">{`${t("form.name.concepts")} (${searchConceptsQuery.data?.totalCount ?? 0})`}</TabsTrigger>
+        <TabsTrigger value="articles">{`${t("taxonomy.resources")} (${
+          searchQuery.data?.totalCount ?? 0
+        })`}</TabsTrigger>
+        <TabsTrigger value="onHold">{`${t("welcomePage.workList.onHold")} (${
+          searchOnHoldQuery.data?.totalCount ?? 0
+        })`}</TabsTrigger>
+        <TabsTrigger value="concepts">{`${t("form.name.concepts")} (${
+          searchConceptsQuery.data?.totalCount ?? 0
+        })`}</TabsTrigger>
         <TabsIndicator />
       </TabsList>
       <WelcomePageTabsContent value="articles">

src/server/translate.ts

@@ -6,6 +6,7 @@
  *
  */
 
+import { CheerioAPI, load } from "cheerio";
 import FormData from "form-data";
 import fetch from "node-fetch";
 import queryString from "query-string";
@@ -41,6 +42,16 @@ const headers = user
     }
   : undefined;
 
+const wrapAttribute = (html: CheerioAPI, element: any, attribute: string, selector: string) => {
+  const value = html(element).attr(attribute) ?? "";
+  if (!value) return;
+  const innerHtml = load(value);
+  innerHtml(selector).each((_, el) => {
+    innerHtml(el).wrap("<ndlaskip></ndlaskip>");
+  });
+  html(element).attr(attribute, innerHtml("body").html());
+};
+
 const doFetch = (name: string, element: ApiTranslateType): Promise<ResponseType> => {
   if (element.type === "text") {
     const parsedContent = element.isArray ? element.content.join("|") : element.content;
@@ -60,8 +71,25 @@ const doFetch = (name: string, element: ApiTranslateType): Promise<ResponseType>
       });
   } else {
     const formData = new FormData();
-    const wrappedContent = `<html>${element.content}</html>`;
-    const buffer = Buffer.from(wrappedContent);
+    const html = load(`${element.content}`);
+    html("span[lang]").each((_, el) => {
+      html(el).wrap("<ndlaskip></ndlaskip>");
+    });
+    html("math").each((_, el) => {
+      html(el).wrap("<ndlaskip></ndlaskip>");
+    });
+    html("ndlaembed").each((_, el) => {
+      wrapAttribute(html, el, "data-caption", "span[lang]");
+      wrapAttribute(html, el, "data-title", "span[lang]");
+      wrapAttribute(html, el, "data-subtitle", "span[lang]");
+      wrapAttribute(html, el, "data-description", "span[lang]");
+      wrapAttribute(html, el, "data-url-text", "span[lang]");
+    });
+    // Our backend uses Jsoup to encode html. However, nynodata expects it to be not encoded. As such, we have to parse
+    // the entire html string and reencode it.
+    const content = html.html({ xml: { xmlMode: false, decodeEntities: false } });
+
+    const buffer = Buffer.from(content);
     const params = { stilmal };
 
     formData.append("file", buffer, { filename: `${name}.html` });
@@ -73,7 +101,11 @@ const doFetch = (name: string, element: ApiTranslateType): Promise<ResponseType>
       .then((res) => res.blob())
       .then((res) => res.text())
       .then(async (res) => {
-        const strippedResponse = res.replace("<html>", "").replace("</html>", "");
+        const response = load(res);
+        response("ndlaskip").each((_, el) => {
+          response(el).contents().unwrap();
+        });
+        const strippedResponse = response("body").unwrap().html() ?? "";
         return { key: name, value: strippedResponse };
       });
   }

yarn.lock

@@ -4719,6 +4719,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"boolbase@npm:^1.0.0":
+  version: 1.0.0
+  resolution: "boolbase@npm:1.0.0"
+  checksum: 10c0/e4b53deb4f2b85c52be0e21a273f2045c7b6a6ea002b0e139c744cb6f95e9ec044439a52883b0d74dedd1ff3da55ed140cfdddfed7fb0cccbed373de5dce1bcf
+  languageName: node
+  linkType: hard
+
 "bowser@npm:2.9.0":
   version: 2.9.0
   resolution: "bowser@npm:2.9.0"
@@ -4947,6 +4954,39 @@ __metadata:
   languageName: node
   linkType: hard
 
+"cheerio-select@npm:^2.1.0":
+  version: 2.1.0
+  resolution: "cheerio-select@npm:2.1.0"
+  dependencies:
+    boolbase: "npm:^1.0.0"
+    css-select: "npm:^5.1.0"
+    css-what: "npm:^6.1.0"
+    domelementtype: "npm:^2.3.0"
+    domhandler: "npm:^5.0.3"
+    domutils: "npm:^3.0.1"
+  checksum: 10c0/2242097e593919dba4aacb97d7b8275def8b9ec70b00aa1f43335456870cfc9e284eae2080bdc832ed232dabb9eefcf56c722d152da4a154813fb8814a55d282
+  languageName: node
+  linkType: hard
+
+"cheerio@npm:^1.0.0-rc.12":
+  version: 1.0.0
+  resolution: "cheerio@npm:1.0.0"
+  dependencies:
+    cheerio-select: "npm:^2.1.0"
+    dom-serializer: "npm:^2.0.0"
+    domhandler: "npm:^5.0.3"
+    domutils: "npm:^3.1.0"
+    encoding-sniffer: "npm:^0.2.0"
+    htmlparser2: "npm:^9.1.0"
+    parse5: "npm:^7.1.2"
+    parse5-htmlparser2-tree-adapter: "npm:^7.0.0"
+    parse5-parser-stream: "npm:^7.1.2"
+    undici: "npm:^6.19.5"
+    whatwg-mimetype: "npm:^4.0.0"
+  checksum: 10c0/d0e16925d9c36c879edfaef1c0244c866375a4c7b8d6ccd7ae0ad42da7d26263ea1a3c17b9a1aa5965918deeff2d40ac2e7223824f8e6eca972df3b81316a09f
+  languageName: node
+  linkType: hard
+
 "chokidar@npm:3.3.1":
   version: 3.3.1
   resolution: "chokidar@npm:3.3.1"
@@ -5361,6 +5401,26 @@ __metadata:
   languageName: node
   linkType: hard
 
+"css-select@npm:^5.1.0":
+  version: 5.1.0
+  resolution: "css-select@npm:5.1.0"
+  dependencies:
+    boolbase: "npm:^1.0.0"
+    css-what: "npm:^6.1.0"
+    domhandler: "npm:^5.0.2"
+    domutils: "npm:^3.0.1"
+    nth-check: "npm:^2.0.1"
+  checksum: 10c0/551c60dba5b54054741032c1793b5734f6ba45e23ae9e82761a3c0ed1acbb8cfedfa443aaba3a3c1a54cac12b456d2012a09d2cd5f0e82e430454c1b9d84d500
+  languageName: node
+  linkType: hard
+
+"css-what@npm:^6.1.0":
+  version: 6.1.0
+  resolution: "css-what@npm:6.1.0"
+  checksum: 10c0/a09f5a6b14ba8dcf57ae9a59474722e80f20406c53a61e9aedb0eedc693b135113ffe2983f4efc4b5065ae639442e9ae88df24941ef159c218b231011d733746
+  languageName: node
+  linkType: hard
+
 "css.escape@npm:^1.5.1":
   version: 1.5.1
   resolution: "css.escape@npm:1.5.1"
@@ -5733,7 +5793,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"domutils@npm:^3.1.0":
+"domutils@npm:^3.0.1, domutils@npm:^3.1.0":
   version: 3.1.0
   resolution: "domutils@npm:3.1.0"
   dependencies:
@@ -5837,6 +5897,7 @@ __metadata:
     "@types/react-dom": "npm:^19.0.1"
     "@vitejs/plugin-react": "npm:^4.3.4"
     auth0-js: "npm:^9.22.1"
+    cheerio: "npm:^1.0.0-rc.12"
     compression: "npm:^1.7.4"
     concurrently: "npm:^9.1.0"
     cross-env: "npm:^7.0.3"
@@ -5949,6 +6010,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"encoding-sniffer@npm:^0.2.0":
+  version: 0.2.0
+  resolution: "encoding-sniffer@npm:0.2.0"
+  dependencies:
+    iconv-lite: "npm:^0.6.3"
+    whatwg-encoding: "npm:^3.1.1"
+  checksum: 10c0/b312e0d67f339bec44e021e5210ee8ee90d7b8f9975eb2c79a36fd467eb07709e88dcf62ee20f62ee0d74a13874307d99557852a2de9b448f1e3fb991fc68257
+  languageName: node
+  linkType: hard
+
 "encoding@npm:^0.1.13":
   version: 0.1.13
   resolution: "encoding@npm:0.1.13"
@@ -7866,7 +7937,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"htmlparser2@npm:9.1.0":
+"htmlparser2@npm:9.1.0, htmlparser2@npm:^9.1.0":
   version: 9.1.0
   resolution: "htmlparser2@npm:9.1.0"
   dependencies:
@@ -7994,7 +8065,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"iconv-lite@npm:0.6.3, iconv-lite@npm:^0.6.2":
+"iconv-lite@npm:0.6.3, iconv-lite@npm:^0.6.2, iconv-lite@npm:^0.6.3":
   version: 0.6.3
   resolution: "iconv-lite@npm:0.6.3"
   dependencies:
@@ -9645,6 +9716,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"nth-check@npm:^2.0.1":
+  version: 2.1.1
+  resolution: "nth-check@npm:2.1.1"
+  dependencies:
+    boolbase: "npm:^1.0.0"
+  checksum: 10c0/5fee7ff309727763689cfad844d979aedd2204a817fbaaf0e1603794a7c20db28548d7b024692f953557df6ce4a0ee4ae46cd8ebd9b36cfb300b9226b567c479
+  languageName: node
+  linkType: hard
+
 "nwsapi@npm:^2.2.12":
   version: 2.2.13
   resolution: "nwsapi@npm:2.2.13"
@@ -9874,6 +9954,34 @@ __metadata:
   languageName: node
   linkType: hard
 
+"parse5-htmlparser2-tree-adapter@npm:^7.0.0":
+  version: 7.1.0
+  resolution: "parse5-htmlparser2-tree-adapter@npm:7.1.0"
+  dependencies:
+    domhandler: "npm:^5.0.3"
+    parse5: "npm:^7.0.0"
+  checksum: 10c0/e5a4e0b834c84c9e244b5749f8d007f4baaeafac7a1da2c54be3421ffd9ef8fdec4f198bf55cda22e88e6ba95e9943f6ed5aa3ae5900b39972ebf5dc8c3f4722
+  languageName: node
+  linkType: hard
+
+"parse5-parser-stream@npm:^7.1.2":
+  version: 7.1.2
+  resolution: "parse5-parser-stream@npm:7.1.2"
+  dependencies:
+    parse5: "npm:^7.0.0"
+  checksum: 10c0/e236c61000d38ecad369e725a48506b051cebad8abb00e6d4e8bff7aa85c183820fcb45db1559cc90955bdbbdbd665ea94c41259594e74566fff411478dc7fcb
+  languageName: node
+  linkType: hard
+
+"parse5@npm:^7.0.0":
+  version: 7.2.1
+  resolution: "parse5@npm:7.2.1"
+  dependencies:
+    entities: "npm:^4.5.0"
+  checksum: 10c0/829d37a0c709215a887e410a7118d754f8e1afd7edb529db95bc7bbf8045fb0266a7b67801331d8e8d9d073ea75793624ec27ce9ff3b96862c3b9008f4d68e80
+  languageName: node
+  linkType: hard
+
 "parse5@npm:^7.1.2":
   version: 7.1.2
   resolution: "parse5@npm:7.1.2"
@@ -12511,6 +12619,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"undici@npm:^6.19.5":
+  version: 6.21.0
+  resolution: "undici@npm:6.21.0"
+  checksum: 10c0/afa9bde6dcf8e0f5cf1ff2fa977ba73dd5510299ddfca0e1f37ff326554172ae31cb3d4a40b5a729601be1f21b96a2684f974d74dab53f9b6930fd47d1949246
+  languageName: node
+  linkType: hard
+
 "unfetch@npm:^4.2.0":
   version: 4.2.0
   resolution: "unfetch@npm:4.2.0"