From f154c31db6a440e6010ad98a758f2ad832d48313 Mon Sep 17 00:00:00 2001 From: sonic16x Date: Thu, 5 Dec 2024 03:59:04 +0700 Subject: [PATCH] Sort and add g-zip size --- package-lock.json | 25 +++++++++++++++++++++---- package.json | 1 + src/services/dictionary.ts | 27 +++++++++++++++------------ src/services/generateDictionary.ts | 8 +++++++- src/utils/deduplicate/deduplicate.ts | 1 + src/utils/deduplicate/index.ts | 1 + src/utils/index.ts | 1 + 7 files changed, 47 insertions(+), 17 deletions(-) create mode 100644 src/utils/deduplicate/deduplicate.ts create mode 100644 src/utils/deduplicate/index.ts diff --git a/package-lock.json b/package-lock.json index cceb243c..3abc14c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -62,6 +62,7 @@ "eslint-plugin-storybook": "^0.11.1", "file-loader": "^6.2.0", "globals": "^15.12.0", + "gzip-size": "^7.0.0", "html-webpack-plugin": "^5.6.3", "jest": "^29.7.0", "jsdom": "^25.0.1", @@ -11339,16 +11340,16 @@ "license": "MIT" }, "node_modules/gzip-size": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", - "integrity": "sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-7.0.0.tgz", + "integrity": "sha512-O1Ld7Dr+nqPnmGpdhzLmMTQ4vAsD+rHwMm1NLUmoUFFymBOMKxCCrtDxqdBRYXdeEPEi3SyoR4TizJLQrnKBNA==", "dev": true, "license": "MIT", "dependencies": { "duplexer": "^0.1.2" }, "engines": { - "node": ">=10" + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -21141,6 +21142,22 @@ "node": ">= 10" } }, + "node_modules/webpack-bundle-analyzer/node_modules/gzip-size": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", + "integrity": "sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "duplexer": "^0.1.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/webpack-bundle-analyzer/node_modules/ws": { "version": "7.5.10", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", diff --git a/package.json b/package.json index 78db44fd..3cfd4aa5 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,7 @@ "eslint-plugin-storybook": "^0.11.1", "file-loader": "^6.2.0", "globals": "^15.12.0", + "gzip-size": "^7.0.0", "html-webpack-plugin": "^5.6.3", "jest": "^29.7.0", "jsdom": "^25.0.1", diff --git a/src/services/dictionary.ts b/src/services/dictionary.ts index ffda021c..c92866b2 100644 --- a/src/services/dictionary.ts +++ b/src/services/dictionary.ts @@ -4,6 +4,7 @@ import { IAlphabets } from 'reducers' import { convertCases, + deduplicate, filterLatin, filterNiqqud, getCaseTips, @@ -235,20 +236,22 @@ class DictionaryClass { switch (lang) { case 'isv': - splittedField = this - .splittedMap['isv-src'] - .get(id) - .map((word) => this.searchPrepare(lang, word)) - .filter((item, i, ar) => ar.indexOf(item) === i) - + splittedField = deduplicate( + this + .splittedMap['isv-src'] + .get(id) + .map((word) => this.searchPrepare(lang, word)) + ) + break case 'isv-src': - splittedField = this - .splitWords(fromField) - .concat(getWordForms(item)) - .map((word) => this.searchPrepare('isv-src', getLatin(word, '2'))) - .filter((item, i, ar) => ar.indexOf(item) === i) - + splittedField = deduplicate( + this + .splitWords(fromField) + .concat(getWordForms(item)) + .map((word) => this.searchPrepare('isv-src', getLatin(word, '2'))) + ) + break default: splittedField = this.splitWords(fromField).map((word) => this.searchPrepare(lang, word)) diff --git a/src/services/generateDictionary.ts b/src/services/generateDictionary.ts index 84682b22..5e28c065 100644 --- a/src/services/generateDictionary.ts +++ b/src/services/generateDictionary.ts @@ -6,6 +6,8 @@ import { Dictionary, loadTablesData } from 'services' import { getColumnName, transposeMatrix } from 'utils' +import { gzipSizeSync } from 'gzip-size' + loadTablesData.then(({ data, columns }) => { Dictionary.init(data) @@ -24,7 +26,7 @@ loadTablesData.then(({ data, columns }) => { } }) - const basicData = transposeMatrix(basicDataTransposed) + const basicData = transposeMatrix(basicDataTransposed).sort((a, b) => a[1].localeCompare(b[1])) const searchIndexBasic = [ 'isv-src', @@ -49,6 +51,10 @@ loadTablesData.then(({ data, columns }) => { fs.writeFileSync('./static/data/basic.json', jsonDataStr) fs.writeFileSync('./static/data/translateStatistic.json', translateStatisticStr) + const sizeMB = gzipSizeSync(jsonDataStr) / 1000000 + // eslint-disable-next-line no-console + console.info(`basic.json g-zip size: ${sizeMB.toFixed(2)} MB`) + addLangs.forEach((lang) => { const langDataTransposed = [ columns.find(([fieldName]) => fieldName === lang), diff --git a/src/utils/deduplicate/deduplicate.ts b/src/utils/deduplicate/deduplicate.ts new file mode 100644 index 00000000..6e35addc --- /dev/null +++ b/src/utils/deduplicate/deduplicate.ts @@ -0,0 +1 @@ +export const deduplicate = (arr) => Array.from(new Set(arr)) diff --git a/src/utils/deduplicate/index.ts b/src/utils/deduplicate/index.ts new file mode 100644 index 00000000..dc2f319c --- /dev/null +++ b/src/utils/deduplicate/index.ts @@ -0,0 +1 @@ +export * from './deduplicate' diff --git a/src/utils/index.ts b/src/utils/index.ts index 7268e426..efd28cad 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,6 +1,7 @@ export * from './bcp47' export * from './convertCases' export * from './createTaggedTemplate' +export * from './deduplicate' export * from './filterLatin' export * from './filterNiqqud' export * from './getAllDataFromResults'