From ff866878af4da004f03b4872a534747106049a0e Mon Sep 17 00:00:00 2001 From: GedionT Date: Tue, 24 Sep 2024 15:53:42 +0300 Subject: [PATCH 01/31] Feat(pdfjs-dist): library import for client-side text extraction --- package-lock.json | 930 ++++++++++++++++++++++++++++++++++++++++++++-- package.json | 1 + 2 files changed, 891 insertions(+), 40 deletions(-) diff --git a/package-lock.json b/package-lock.json index f991c75..1e9276b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -66,6 +66,7 @@ "lodash.uniq": "^4.5.0", "lodash.uniqby": "^4.7.0", "lucide-react": "^0.289.0", + "pdfjs-dist": "^4.6.82", "react": "^18.2.0", "react-dom": "^18.2.0", "react-intersection-observer": "^8.34.0", @@ -1068,6 +1069,26 @@ "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==" }, + "node_modules/@mapbox/node-pre-gyp": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz", + "integrity": "sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.0", + "https-proxy-agent": "^5.0.0", + "make-dir": "^3.1.0", + "node-fetch": "^2.6.7", + "nopt": "^5.0.0", + "npmlog": "^5.0.1", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.11" + }, + "bin": { + "node-pre-gyp": "bin/node-pre-gyp" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2079,6 +2100,12 @@ "vite": "^4.2.0" } }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "optional": true + }, "node_modules/acorn": { "version": "7.4.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", @@ -2100,6 +2127,18 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "optional": true, + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -2129,7 +2168,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, + "devOptional": true, "engines": { "node": ">=8" } @@ -2235,6 +2274,26 @@ "react-dom": ">=16.0.0" } }, + "node_modules/aproba": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", + "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==", + "optional": true + }, + "node_modules/are-we-there-yet": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz", + "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==", + "deprecated": "This package is no longer supported.", + "optional": true, + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -2437,13 +2496,13 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "devOptional": true }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, + "devOptional": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2539,6 +2598,21 @@ } ] }, + "node_modules/canvas": { + "version": "2.11.2", + "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz", + "integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "@mapbox/node-pre-gyp": "^1.0.0", + "nan": "^2.17.0", + "simple-get": "^3.0.3" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -2552,6 +2626,15 @@ "node": ">=4" } }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "optional": true, + "engines": { + "node": ">=10" + } + }, "node_modules/classcat": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.4.tgz", @@ -2575,6 +2658,15 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" }, + "node_modules/color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "optional": true, + "bin": { + "color-support": "bin.js" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -2600,7 +2692,7 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true + "devOptional": true }, "node_modules/confusing-browser-globals": { "version": "1.0.11", @@ -2608,6 +2700,12 @@ "integrity": "sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA==", "dev": true }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", + "optional": true + }, "node_modules/consolidated-events": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/consolidated-events/-/consolidated-events-2.0.2.tgz", @@ -2941,6 +3039,18 @@ } } }, + "node_modules/decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "optional": true, + "dependencies": { + "mimic-response": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/deep-equal": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.1.tgz", @@ -3000,6 +3110,21 @@ "node": ">=0.4.0" } }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "optional": true + }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -3044,7 +3169,7 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "devOptional": true }, "node_modules/enquirer": { "version": "2.3.6", @@ -3956,11 +4081,35 @@ "node": ">= 6" } }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "devOptional": true }, "node_modules/fsevents": { "version": "2.3.2", @@ -4015,6 +4164,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gauge": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz", + "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==", + "deprecated": "This package is no longer supported.", + "optional": true, + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.2", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.1", + "object-assign": "^4.1.1", + "signal-exit": "^3.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.2" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -4059,7 +4229,7 @@ "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dev": true, + "devOptional": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -4228,6 +4398,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", + "optional": true + }, "node_modules/hoist-non-react-statics": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", @@ -4236,6 +4412,19 @@ "react-is": "^16.7.0" } }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "optional": true, + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -4285,7 +4474,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, + "devOptional": true, "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -4295,7 +4484,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "devOptional": true }, "node_modules/internal-slot": { "version": "1.0.5", @@ -4429,7 +4618,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, + "devOptional": true, "engines": { "node": ">=8" } @@ -4817,7 +5006,7 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, + "devOptional": true, "dependencies": { "yallist": "^4.0.0" }, @@ -4833,6 +5022,30 @@ "react": "^16.5.1 || ^17.0.0 || ^18.0.0" } }, + "node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "optional": true, + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "optional": true, + "bin": { + "semver": "bin/semver.js" + } + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -4874,11 +5087,23 @@ "node": ">= 0.6" } }, + "node_modules/mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", + "optional": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, + "devOptional": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -4895,11 +5120,63 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "optional": true, + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "optional": true, + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/nan": { + "version": "2.20.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.20.0.tgz", + "integrity": "sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==", + "optional": true + }, "node_modules/nanoid": { "version": "3.3.6", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", @@ -4930,12 +5207,60 @@ "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==", "dev": true }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "optional": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-releases": { "version": "2.0.12", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.12.tgz", "integrity": "sha512-QzsYKWhXTWx8h1kIvqfnC++o0pEmpRQA/aenALsL2F4pqNVr7YzcdMlDij5WBnwftRbJCNJL/O7zdKaxKPHqgQ==", "dev": true }, + "node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "optional": true, + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/npmlog": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz", + "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==", + "deprecated": "This package is no longer supported.", + "optional": true, + "dependencies": { + "are-we-there-yet": "^2.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^3.0.0", + "set-blocking": "^2.0.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -5061,7 +5386,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, + "devOptional": true, "dependencies": { "wrappy": "1" } @@ -5099,7 +5424,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, + "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -5128,6 +5453,27 @@ "node": ">=8" } }, + "node_modules/path2d": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/path2d/-/path2d-0.2.1.tgz", + "integrity": "sha512-Fl2z/BHvkTNvkuBzYTpTuirHZg6wW9z8+4SND/3mDTEcYbbNKWAy21dz9D3ePNNwrrK8pqZO5vLPZ1hLF6T7XA==", + "optional": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/pdfjs-dist": { + "version": "4.6.82", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-4.6.82.tgz", + "integrity": "sha512-BUOryeRFwvbLe0lOU6NhkJNuVQUp06WxlJVVCsxdmJ4y5cU3O3s3/0DunVdK1PMm7v2MUw52qKYaidhDH1Z9+w==", + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "canvas": "^2.11.2", + "path2d": "^0.2.1" + } + }, "node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -6005,6 +6351,20 @@ "react-dom": ">=17" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "optional": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/regenerator-runtime": { "version": "0.13.11", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", @@ -6093,7 +6453,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "dev": true, + "devOptional": true, "dependencies": { "glob": "^7.1.3" }, @@ -6148,6 +6508,26 @@ "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==" }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true + }, "node_modules/safe-regex-test": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz", @@ -6187,7 +6567,7 @@ "version": "7.5.1", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.1.tgz", "integrity": "sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==", - "dev": true, + "devOptional": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -6198,6 +6578,12 @@ "node": ">=10" } }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "optional": true + }, "node_modules/shallowequal": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz", @@ -6238,6 +6624,43 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "optional": true + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true + }, + "node_modules/simple-get": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz", + "integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==", + "optional": true, + "dependencies": { + "decompress-response": "^4.2.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -6324,6 +6747,15 @@ "node": ">= 0.4" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "optional": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-convert": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz", @@ -6333,7 +6765,7 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, + "devOptional": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -6411,7 +6843,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, + "devOptional": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -6535,6 +6967,23 @@ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true }, + "node_modules/tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "optional": true, + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -6574,6 +7023,12 @@ "resolved": "https://registry.npmjs.org/toggle-selection/-/toggle-selection-1.0.6.tgz", "integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==" }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "optional": true + }, "node_modules/tsconfig-paths": { "version": "3.14.2", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz", @@ -6736,6 +7191,12 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "optional": true + }, "node_modules/v8-compile-cache": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", @@ -6825,6 +7286,22 @@ "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-1.1.2.tgz", "integrity": "sha512-PFMKIY+bRSXlMxVAQ+m2aw9c/ioUYfDgrYot0YUa+/xa0sakubWhSDyxAKwzymvXVdF4CZI71g06W+mqhzu6ig==" }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "optional": true + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "optional": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -6891,6 +7368,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "optional": true, + "dependencies": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, "node_modules/word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -6904,7 +7390,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "devOptional": true }, "node_modules/xmlhttprequest": { "version": "1.8.0", @@ -6918,7 +7404,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "devOptional": true }, "node_modules/zustand": { "version": "4.3.9", @@ -7574,6 +8060,23 @@ } } }, + "@mapbox/node-pre-gyp": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz", + "integrity": "sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==", + "optional": true, + "requires": { + "detect-libc": "^2.0.0", + "https-proxy-agent": "^5.0.0", + "make-dir": "^3.1.0", + "node-fetch": "^2.6.7", + "nopt": "^5.0.0", + "npmlog": "^5.0.1", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.11" + } + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -8390,6 +8893,12 @@ "react-refresh": "^0.14.0" } }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "optional": true + }, "acorn": { "version": "7.4.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", @@ -8403,6 +8912,15 @@ "dev": true, "requires": {} }, + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "optional": true, + "requires": { + "debug": "4" + } + }, "ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -8425,7 +8943,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true + "devOptional": true }, "ansi-styles": { "version": "3.2.1", @@ -8512,6 +9030,22 @@ } } }, + "aproba": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", + "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==", + "optional": true + }, + "are-we-there-yet": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz", + "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==", + "optional": true, + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + } + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -8675,13 +9209,13 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "devOptional": true }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, + "devOptional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -8735,6 +9269,17 @@ "integrity": "sha512-x1mgZEXK8jHIfAxm+xgdpHpk50IN3z3q3zP261/WS+uvePxW8izXuCu6AHz0lkuYTlATDehiZ/tNyYBdSQsOUQ==", "dev": true }, + "canvas": { + "version": "2.11.2", + "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz", + "integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==", + "optional": true, + "requires": { + "@mapbox/node-pre-gyp": "^1.0.0", + "nan": "^2.17.0", + "simple-get": "^3.0.3" + } + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -8745,6 +9290,12 @@ "supports-color": "^5.3.0" } }, + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "optional": true + }, "classcat": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.4.tgz", @@ -8768,6 +9319,12 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" }, + "color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "optional": true + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -8790,7 +9347,7 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true + "devOptional": true }, "confusing-browser-globals": { "version": "1.0.11", @@ -8798,6 +9355,12 @@ "integrity": "sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA==", "dev": true }, + "console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", + "optional": true + }, "consolidated-events": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/consolidated-events/-/consolidated-events-2.0.2.tgz", @@ -9048,6 +9611,15 @@ "ms": "2.1.2" } }, + "decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "optional": true, + "requires": { + "mimic-response": "^2.0.0" + } + }, "deep-equal": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.1.tgz", @@ -9095,6 +9667,18 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, + "delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "optional": true + }, + "detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "optional": true + }, "dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -9133,7 +9717,7 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "devOptional": true }, "enquirer": { "version": "2.3.6", @@ -9833,11 +10417,31 @@ "mime-types": "^2.1.12" } }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "optional": true, + "requires": { + "minipass": "^3.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "optional": true, + "requires": { + "yallist": "^4.0.0" + } + } + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "devOptional": true }, "fsevents": { "version": "2.3.2", @@ -9876,6 +10480,23 @@ "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", "dev": true }, + "gauge": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz", + "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==", + "optional": true, + "requires": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.2", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.1", + "object-assign": "^4.1.1", + "signal-exit": "^3.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.2" + } + }, "gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -9908,7 +10529,7 @@ "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dev": true, + "devOptional": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -10020,6 +10641,12 @@ "has-symbols": "^1.0.2" } }, + "has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", + "optional": true + }, "hoist-non-react-statics": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", @@ -10028,6 +10655,16 @@ "react-is": "^16.7.0" } }, + "https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "optional": true, + "requires": { + "agent-base": "6", + "debug": "4" + } + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -10062,7 +10699,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, + "devOptional": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -10072,7 +10709,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "devOptional": true }, "internal-slot": { "version": "1.0.5", @@ -10164,7 +10801,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true + "devOptional": true }, "is-glob": { "version": "4.0.3", @@ -10465,7 +11102,7 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, + "devOptional": true, "requires": { "yallist": "^4.0.0" } @@ -10476,6 +11113,23 @@ "integrity": "sha512-D3/kt5h4KVmO9Bqlhky/szWI3puEU/KJfQWCeX8Zhvx3xx0SQ4t6vbwiK9ORBbiaqXefkBbXjoq7fOBd7s5yXQ==", "requires": {} }, + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "optional": true, + "requires": { + "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "optional": true + } + } + }, "merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -10505,11 +11159,17 @@ "mime-db": "1.52.0" } }, + "mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", + "optional": true + }, "minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, + "devOptional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -10520,11 +11180,50 @@ "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", "dev": true }, + "minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "optional": true + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "optional": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "optional": true, + "requires": { + "yallist": "^4.0.0" + } + } + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "optional": true + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "nan": { + "version": "2.20.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.20.0.tgz", + "integrity": "sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==", + "optional": true + }, "nanoid": { "version": "3.3.6", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", @@ -10543,12 +11242,42 @@ "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==", "dev": true }, + "node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "optional": true, + "requires": { + "whatwg-url": "^5.0.0" + } + }, "node-releases": { "version": "2.0.12", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.12.tgz", "integrity": "sha512-QzsYKWhXTWx8h1kIvqfnC++o0pEmpRQA/aenALsL2F4pqNVr7YzcdMlDij5WBnwftRbJCNJL/O7zdKaxKPHqgQ==", "dev": true }, + "nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "optional": true, + "requires": { + "abbrev": "1" + } + }, + "npmlog": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz", + "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==", + "optional": true, + "requires": { + "are-we-there-yet": "^2.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^3.0.0", + "set-blocking": "^2.0.0" + } + }, "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -10635,7 +11364,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, + "devOptional": true, "requires": { "wrappy": "1" } @@ -10667,7 +11396,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true + "devOptional": true }, "path-key": { "version": "3.1.1", @@ -10687,6 +11416,21 @@ "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", "dev": true }, + "path2d": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/path2d/-/path2d-0.2.1.tgz", + "integrity": "sha512-Fl2z/BHvkTNvkuBzYTpTuirHZg6wW9z8+4SND/3mDTEcYbbNKWAy21dz9D3ePNNwrrK8pqZO5vLPZ1hLF6T7XA==", + "optional": true + }, + "pdfjs-dist": { + "version": "4.6.82", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-4.6.82.tgz", + "integrity": "sha512-BUOryeRFwvbLe0lOU6NhkJNuVQUp06WxlJVVCsxdmJ4y5cU3O3s3/0DunVdK1PMm7v2MUw52qKYaidhDH1Z9+w==", + "requires": { + "canvas": "^2.11.2", + "path2d": "^0.2.1" + } + }, "picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -11275,6 +12019,17 @@ "@reactflow/node-toolbar": "1.2.3" } }, + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "optional": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, "regenerator-runtime": { "version": "0.13.11", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", @@ -11335,7 +12090,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "dev": true, + "devOptional": true, "requires": { "glob": "^7.1.3" } @@ -11363,6 +12118,12 @@ "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==" }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "optional": true + }, "safe-regex-test": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz", @@ -11399,11 +12160,17 @@ "version": "7.5.1", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.1.tgz", "integrity": "sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==", - "dev": true, + "devOptional": true, "requires": { "lru-cache": "^6.0.0" } }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "optional": true + }, "shallowequal": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz", @@ -11435,6 +12202,29 @@ "object-inspect": "^1.9.0" } }, + "signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "optional": true + }, + "simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "optional": true + }, + "simple-get": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz", + "integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==", + "optional": true, + "requires": { + "decompress-response": "^4.2.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -11499,6 +12289,15 @@ "internal-slot": "^1.0.4" } }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "optional": true, + "requires": { + "safe-buffer": "~5.2.0" + } + }, "string-convert": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz", @@ -11508,7 +12307,7 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, + "devOptional": true, "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -11568,7 +12367,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, + "devOptional": true, "requires": { "ansi-regex": "^5.0.1" } @@ -11654,6 +12453,20 @@ } } }, + "tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "optional": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -11684,6 +12497,12 @@ "resolved": "https://registry.npmjs.org/toggle-selection/-/toggle-selection-1.0.6.tgz", "integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==" }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "optional": true + }, "tsconfig-paths": { "version": "3.14.2", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz", @@ -11795,6 +12614,12 @@ "integrity": "sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==", "requires": {} }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "optional": true + }, "v8-compile-cache": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", @@ -11840,6 +12665,22 @@ "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-1.1.2.tgz", "integrity": "sha512-PFMKIY+bRSXlMxVAQ+m2aw9c/ioUYfDgrYot0YUa+/xa0sakubWhSDyxAKwzymvXVdF4CZI71g06W+mqhzu6ig==" }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "optional": true + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "optional": true, + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -11888,6 +12729,15 @@ "is-typed-array": "^1.1.10" } }, + "wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "optional": true, + "requires": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -11898,7 +12748,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "devOptional": true }, "xmlhttprequest": { "version": "1.8.0", @@ -11909,7 +12759,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "devOptional": true }, "zustand": { "version": "4.3.9", diff --git a/package.json b/package.json index 4a089db..c7cf24b 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "lodash.uniq": "^4.5.0", "lodash.uniqby": "^4.7.0", "lucide-react": "^0.289.0", + "pdfjs-dist": "^4.6.82", "react": "^18.2.0", "react-dom": "^18.2.0", "react-intersection-observer": "^8.34.0", From 7932c857863021eb85947f3626227e6565d8ed51 Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 13:37:16 +0300 Subject: [PATCH 02/31] Feat(api): scaffolded api calls for priorities --- src/api/prioritiesCall.ts | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/api/prioritiesCall.ts diff --git a/src/api/prioritiesCall.ts b/src/api/prioritiesCall.ts new file mode 100644 index 0000000..0812e1b --- /dev/null +++ b/src/api/prioritiesCall.ts @@ -0,0 +1,66 @@ +import axios from 'axios'; +import { + DIAGNOSTICS_API_BASE_URL, + DIAGNOSTICS_API_ACCESS_TOKEN, +} from '../Constants'; + +export const fetchMetadata = async ( + iso: string, + year?: number, + language?: string, + kind?: string, +) => { + const params = { + iso: iso.toLowerCase(), + year, + language, + kind, + }; + const response = await axios.get(`${DIAGNOSTICS_API_BASE_URL}/metadata`, { + params, + headers: { + accept: 'application/json', + api_key: DIAGNOSTICS_API_ACCESS_TOKEN, + }, + }); + return response.data; +}; + +export const fetchDocumentById = async (id: string) => { + const response = await axios.get( + `${DIAGNOSTICS_API_BASE_URL}/documents/${id}`, + { + headers: { + accept: 'application/json', + api_key: DIAGNOSTICS_API_ACCESS_TOKEN, + }, + }, + ); + return response.data; +}; + +export const submitDocumentsForAnalysis = async ( + files: File[], + weights?: number[], +) => { + const formData = new FormData(); + + files.forEach((file) => formData.append('files', file)); + + const weightParam = weights && weights.length > 0 + ? `?${weights.map((weight) => `weights=${weight}`).join('&')}` + : ''; + + const response = await axios.post( + `${DIAGNOSTICS_API_BASE_URL}/documents${weightParam}`, + formData, + { + headers: { + 'Content-Type': 'multipart/form-data', + api_key: DIAGNOSTICS_API_ACCESS_TOKEN, + }, + }, + ); + + return response.data; +}; From 0ea144964bf910948c27ef6cbe4f1edceebc9abe Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 13:41:31 +0300 Subject: [PATCH 03/31] Feat(util): added a client-side text extraction util with fallback mechanism via api --- src/utils/extractPDF.ts | 171 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 src/utils/extractPDF.ts diff --git a/src/utils/extractPDF.ts b/src/utils/extractPDF.ts new file mode 100644 index 0000000..9bd2816 --- /dev/null +++ b/src/utils/extractPDF.ts @@ -0,0 +1,171 @@ +import axios from 'axios'; +import { + GlobalWorkerOptions, + getDocument, + PDFDocumentProxy, + PDFPageProxy, +} from 'pdfjs-dist'; + +import { + LANGUAGE_DETECTION_API_BASE_URL, + LANGUAGE_DETECTION_API_KEY, + PDFJS_DIST_CDNS, + TEXT_EXTRACTION_API_BASE_URL, + TEXT_EXTRACTION_API_KEY, +} from '../Constants'; + +GlobalWorkerOptions.workerSrc = ''; + +async function checkCdn(cdn: string, index: number) { + try { + const response = await fetch(cdn, { method: 'HEAD' }); + if (response.ok) { + return cdn; + } + console.error( + `CDN ${index + 1} down or invalid MIME type: ${response.headers.get( + 'Content-Type', + )}`, + ); + } catch (error) { + console.error(`Error checking CDN ${index + 1}:`, error); + } + return null; +} + +async function loadPdfWorker() { + const validCdn = await PDFJS_DIST_CDNS.reduce( + async (accPromise, cdn, index) => { + const acc = await accPromise; + if (acc) return acc; + + return checkCdn(cdn, index); + }, + Promise.resolve(null as string | null), + ); + + if (validCdn) { + GlobalWorkerOptions.workerSrc = validCdn; + } else { + throw new Error('All CDNs failed'); + } +} + +async function detectLanguage(text: string) { + try { + const response = await axios({ + method: 'post', + url: LANGUAGE_DETECTION_API_BASE_URL, + data: JSON.stringify({ text }), + headers: { + 'Content-Type': 'application/json', + api_key: LANGUAGE_DETECTION_API_KEY, + }, + }); + + return response.data.official; + } catch (err: any) { + throw new Error(`Language detection API failed: ${err.message}`); + } +} + +async function extractViaAPI(fileData: File) { + const formData = new FormData(); + formData.append('file', fileData); + + try { + const response = await axios({ + method: 'post', + url: TEXT_EXTRACTION_API_BASE_URL, + data: formData, + headers: { + 'Content-Type': 'multipart/form-data', + accept: 'application/json', + api_key: TEXT_EXTRACTION_API_KEY, + }, + }); + + return [ + response.data.pages.map((page: any) => page.text).join(' '), + response.data.pages.length, + ]; + } catch (err: any) { + throw new Error(`Text extraction failed ${err}`); + } +} + +async function pdfToText(file: File) { + const blobUrl = URL.createObjectURL(file); + let extractedText = ''; + let pageCount = 0; + let hadParsingError = false; + + try { + const loadingTask = getDocument(blobUrl); + const pdf: PDFDocumentProxy = await loadingTask.promise; + const { numPages } = pdf; + pageCount = numPages; + + for (let pageNumber = 1; pageNumber <= numPages; pageNumber += 1) { + // eslint-disable-next-line no-await-in-loop + const page: PDFPageProxy = await pdf.getPage(pageNumber); + // eslint-disable-next-line no-await-in-loop + const textContent = await page.getTextContent(); + const pageText = textContent.items + .map((item) => ('str' in item ? item.str : '')) + .join(' '); + extractedText += `${pageText} \n`; + } + + URL.revokeObjectURL(blobUrl); + loadingTask.destroy(); + } catch (error) { + hadParsingError = true; + } + + if (hadParsingError) { + try { + const [extractedTextViaAPI, pageCountViaAPI] = await extractViaAPI(file); + extractedText = extractedTextViaAPI; + pageCount = pageCountViaAPI; + } catch (error: any) { + throw new Error( + `Both PDF parsing and API extraction failed: ${error.message}`, + ); + } + } + + const language = await detectLanguage(extractedText.trim()); + if (language !== 'en') { + throw new Error(`Non-English document detected: ${file.name}`); + } + + return [extractedText.trim(), pageCount]; +} + +export async function extractTextFromMultiplePdfs(files: File[]) { + await loadPdfWorker(); + + const texts = await Promise.all( + files.map(async (file) => { + try { + const [text, pageCount] = await pdfToText(file); + return { + file_name: file.name, + text, + pageCount, + error: false, + }; + } catch (error: any) { + return { + file_name: file.name, + text: error?.message, + pageCount: 0, + error: true, + }; + } + }), + ); + + return texts; +} From fe00a48f6bcd8ba252e97e705cd66fa1f2fc6203 Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 13:42:44 +0300 Subject: [PATCH 04/31] Refactor(saliencGraph): fixed data attribute from d.sdg to d.id --- src/Priorities/SalienceGraph.tsx | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/Priorities/SalienceGraph.tsx b/src/Priorities/SalienceGraph.tsx index 5f45c2a..18ffd96 100644 --- a/src/Priorities/SalienceGraph.tsx +++ b/src/Priorities/SalienceGraph.tsx @@ -16,6 +16,7 @@ export const SalienceGraph = (props: Props) => { data, goalStatuses, } = props; + return ( <>
@@ -27,19 +28,19 @@ export const SalienceGraph = (props: Props) => {

- +

On track

- +

For review

- +

Off Track

- +

Trend NA

@@ -104,7 +105,7 @@ export const SalienceGraph = (props: Props) => { { - data.map((d:any, i: number) => ( + data.map((d: any, i: number) => ( { r={15} cy={400 - (375 * d.importance)} style={{ - fill: goalStatuses.findIndex((el) => el.goal === d.sdg) !== -1 ? goalStatuses[goalStatuses.findIndex((el) => el.goal === d.sdg)].status === 'On Track' ? 'var(--dark-green)' : goalStatuses[goalStatuses.findIndex((el) => el.goal === d.sdg)].status === 'For Review' ? 'var(--dark-yellow)' : goalStatuses[goalStatuses.findIndex((el) => el.goal === d.sdg)].status === 'Identified Gap' ? 'var(--dark-red)' : 'var(--gray-500)' : 'var(--gray-500)', + fill: goalStatuses.findIndex((el) => el.goal === d.id) !== -1 ? goalStatuses[goalStatuses.findIndex((el) => el.goal === d.id)].status === 'On Track' ? 'var(--dark-green)' : goalStatuses[goalStatuses.findIndex((el) => el.goal === d.id)].status === 'For Review' ? 'var(--dark-yellow)' : goalStatuses[goalStatuses.findIndex((el) => el.goal === d.id)].status === 'Identified Gap' ? 'var(--dark-red)' : 'var(--gray-500)' : 'var(--gray-500)', }} /> { y1={400 - (375 * d.importance)} y2={400} style={{ - stroke: goalStatuses.findIndex((el) => el.goal === d.sdg) !== -1 ? goalStatuses[goalStatuses.findIndex((el) => el.goal === d.sdg)].status === 'On Track' ? 'var(--dark-green)' : goalStatuses[goalStatuses.findIndex((el) => el.goal === d.sdg)].status === 'For Review' ? 'var(--dark-yellow)' : goalStatuses[goalStatuses.findIndex((el) => el.goal === d.sdg)].status === 'Identified Gap' ? 'var(--dark-red)' : 'var(--gray-500)' : 'var(--gray-500)', + stroke: goalStatuses.findIndex((el) => el.goal === d.id) !== -1 ? goalStatuses[goalStatuses.findIndex((el) => el.goal === d.id)].status === 'On Track' ? 'var(--dark-green)' : goalStatuses[goalStatuses.findIndex((el) => el.goal === d.id)].status === 'For Review' ? 'var(--dark-yellow)' : goalStatuses[goalStatuses.findIndex((el) => el.goal === d.id)].status === 'Identified Gap' ? 'var(--dark-red)' : 'var(--gray-500)' : 'var(--gray-500)', }} /> { y={400 - (375 * d.importance)} dy={-20} style={{ - fill: goalStatuses.findIndex((el) => el.goal === d.sdg) !== -1 ? goalStatuses[goalStatuses.findIndex((el) => el.goal === d.sdg)].status === 'On Track' ? 'var(--dark-green)' : goalStatuses[goalStatuses.findIndex((el) => el.goal === d.sdg)].status === 'For Review' ? 'var(--dark-yellow)' : goalStatuses[goalStatuses.findIndex((el) => el.goal === d.sdg)].status === 'Identified Gap' ? 'var(--dark-red)' : 'var(--gray-500)' : 'var(--gray-500)', + fill: goalStatuses.findIndex((el) => el.goal === d.id) !== -1 ? goalStatuses[goalStatuses.findIndex((el) => el.goal === d.id)].status === 'On Track' ? 'var(--dark-green)' : goalStatuses[goalStatuses.findIndex((el) => el.goal === d.id)].status === 'For Review' ? 'var(--dark-yellow)' : goalStatuses[goalStatuses.findIndex((el) => el.goal === d.id)].status === 'Identified Gap' ? 'var(--dark-red)' : 'var(--gray-500)' : 'var(--gray-500)', }} fontSize={12} textAnchor='middle' @@ -150,11 +151,8 @@ export const SalienceGraph = (props: Props) => { > SDG {' '} - {d.sdg} + {d.id} - { - - } )) } From 42cc6b03c00d44c25be02535a7349fe6a762dae8 Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 13:43:14 +0300 Subject: [PATCH 05/31] Refactor(BubbleChart): refactored attribute from d.sdg to d.id --- src/Priorities/BubbleChart.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Priorities/BubbleChart.tsx b/src/Priorities/BubbleChart.tsx index 6b3bbf7..f0fbaef 100644 --- a/src/Priorities/BubbleChart.tsx +++ b/src/Priorities/BubbleChart.tsx @@ -39,8 +39,8 @@ export const BubbleChart = (props: Props) => { forceSimulation(dataTemp) .force('charge', forceManyBody().strength(2.25)) .force('center', forceCenter()) - .force('x', forceX().strength(1).x(graphOrientation === 'horizontal' ? (d: any) => (d.sdg * 15) : 0)) - .force('y', forceY().strength(1).y(graphOrientation === 'vertical' ? (d: any) => (d.sdg * 15) : 0)) + .force('x', forceX().strength(1).x(graphOrientation === 'horizontal' ? (d: any) => (d.id * 15) : 0)) + .force('y', forceY().strength(1).y(graphOrientation === 'vertical' ? (d: any) => (d.id * 15) : 0)) .force('collision', forceCollide().radius((d: any) => (d.importance * 50) + 4)) .tick(10000) .on('end', () => { setNodeData(dataTemp); }); @@ -182,13 +182,13 @@ export const BubbleChart = (props: Props) => { cx={0} cy={0} r={d.importance * 50} - fill={SDG_COLOR_ARRAY[d.sdg - 1]} + fill={SDG_COLOR_ARRAY[d.id - 1]} /> { d.importance * 50 < 10 ? null : ( - {getSDGIconSVG(`SDG ${d.sdg}`, d.importance * 70)} + {getSDGIconSVG(`SDG ${d.id}`, d.importance * 70)} ) } From c68be9c55bdad1b293d991ba19b3f849aab0f8ed Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 13:43:39 +0300 Subject: [PATCH 06/31] Refactor(CompareTable): fixed attribute from d.sdg to d.id --- src/Priorities/CompareTable.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Priorities/CompareTable.tsx b/src/Priorities/CompareTable.tsx index 7c6cdd4..f075e2b 100644 --- a/src/Priorities/CompareTable.tsx +++ b/src/Priorities/CompareTable.tsx @@ -19,8 +19,8 @@ export const CompareAnalysis = (props: Props) => { const { data, goalStatuses, document, } = props; - const dataDoc1WithStatuses = data[0].map((d: any) => ({ ...d, status: goalStatuses[goalStatuses.findIndex((el) => el.goal === d.sdg)].status || 'Gaps NA' })); - const dataDoc2WithStatuses = data[1].map((d: any) => ({ ...d, status: goalStatuses[goalStatuses.findIndex((el) => el.goal === d.sdg)].status || 'Gaps NA' })); + const dataDoc1WithStatuses = data[0].map((d: any) => ({ ...d, status: goalStatuses[goalStatuses.findIndex((el) => el.goal === d.id)].status || 'Gaps NA' })); + const dataDoc2WithStatuses = data[1].map((d: any) => ({ ...d, status: goalStatuses[goalStatuses.findIndex((el) => el.goal === d.id)].status || 'Gaps NA' })); let similar = 0; for (let i = 0; i < dataDoc1WithStatuses.length; i += 1) { if (dataDoc1WithStatuses[i].category === dataDoc2WithStatuses[i].category) similar += 1; @@ -72,15 +72,15 @@ export const CompareAnalysis = (props: Props) => {
- {getSDGIcon(`SDG ${d.sdg}`, 72)} + {getSDGIcon(`SDG ${d.id}`, 72)}
SDG {' '} - {d.sdg} + {d.id} : {' '} - {SDGGoalList[SDGGoalList.findIndex((el) => el.Goal === `SDG ${d.sdg}`)]['Goal Name']} + {SDGGoalList[SDGGoalList.findIndex((el) => el.Goal === `SDG ${d.id}`)]['Goal Name']}
{d.status} From 42130a3477a8203e1abfc72449444c60e7a3f24e Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 13:44:36 +0300 Subject: [PATCH 07/31] Refactor(VNRAnalysis): fixed attrib from d.sdg to d.id and code formatting applied --- src/Priorities/VNRAnalysis.tsx | 48 ++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/Priorities/VNRAnalysis.tsx b/src/Priorities/VNRAnalysis.tsx index adb4b47..1f1faa0 100644 --- a/src/Priorities/VNRAnalysis.tsx +++ b/src/Priorities/VNRAnalysis.tsx @@ -75,7 +75,14 @@ export const VNRAnalysis = (props: Props) => { const [hoveredSDG, setHoveredSDG] = useState(null); const [nodeData, setNodeData] = useState(null); const [showSalienceGraph, setShowSalienceGraph] = useState(false); - const dataWithStatuses = data.map((d: any) => ({ ...d, category: d.importance === 0 ? 'No Mention' : d.category.charAt(0).toUpperCase() + d.category.slice(1), status: goalStatuses[goalStatuses.findIndex((el) => el.goal === d.sdg)].status || 'Gaps NA' })); + const dataWithStatuses = data.map((d: any) => { + const goalStatus = goalStatuses.find((el) => el.goal === d.id); + return { + ...d, + category: d.importance === 0 ? 'No Mention' : d.category.charAt(0).toUpperCase() + d.category.slice(1), + status: goalStatus ? goalStatus.status : 'Gaps NA', + }; + }); const gridSize = 600; const margin = 20; const cellSize = (gridSize - margin) / 4; @@ -93,6 +100,7 @@ export const VNRAnalysis = (props: Props) => { .tick(100) .on('end', () => { setNodeData(dataTemp); }); }, [document]); + return ( <>
@@ -103,9 +111,9 @@ export const VNRAnalysis = (props: Props) => { {defaultDocs ? ( <> - {document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} + {document && document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} - ) : document.map((d: any, i: number) => {d})} + ) : document && document.map((d: any, i: number) => {d})}
@@ -133,7 +141,7 @@ export const VNRAnalysis = (props: Props) => { fontWeight='bold' style={{ fill: `${d === 'High' ? 'var(--blue-700)' : d === 'Medium' ? 'var(--blue-400)' : d === 'Low' ? 'var(--blue-200)' : 'var(--gray-400)'}` }} > - { d === 'No Mention'.toUpperCase() ? d : `${d} Priority`.toUpperCase()} + {d === 'No Mention'.toUpperCase() ? d : `${d} Priority`.toUpperCase()} )) } @@ -238,7 +246,7 @@ export const VNRAnalysis = (props: Props) => { style={{ cursor: 'default' }} onMouseEnter={(event) => { setHoveredSDG({ - sdg: d.sdg, + sdg: d.id, xPosition: event.clientX, yPosition: event.clientY, }); @@ -249,7 +257,7 @@ export const VNRAnalysis = (props: Props) => { cx={0} cy={0} r={nodeRadius} - fill={SDG_COLOR_ARRAY[d.sdg - 1]} + fill={SDG_COLOR_ARRAY[d.id - 1]} /> { textAnchor='middle' fill='#fff' > - {d.sdg} + {d.id} )) @@ -284,9 +292,9 @@ export const VNRAnalysis = (props: Props) => { {defaultDocs ? ( <> - {document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} + {document && document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} - ) : document.map((d: any, i: number) => {d})} + ) : document && document.map((d: any, i: number) => {d})}

This matrix maps the SDGs along two parameters @@ -302,11 +310,11 @@ export const VNRAnalysis = (props: Props) => { Disclaimer: The national priorities identified in the documents may not reflect the actual and complete priorities of the government. They are a starting point for further discussion. The SDG Trends assessment is based on currently available data in the {' '} - UN Stats SDG Data Portal + UN Stats SDG Data Portal {' '} and methodology as per the {' '} - SDG Progress Chart 2022 Technical Note + SDG Progress Chart 2022 Technical Note . Additional data may be added to address gaps at government request, to provide a comprehensive landscape for identification of SDG policy pathways. @@ -315,14 +323,14 @@ export const VNRAnalysis = (props: Props) => {

{ - showSalienceGraph - ? ( - - ) : null - } + showSalienceGraph + ? ( + + ) : null + }
@@ -348,7 +356,7 @@ export const VNRAnalysis = (props: Props) => { className='undp-modal' onCancel={() => { setSelectedSDG(null); }} onOk={() => { setSelectedSDG(null); }} - title={selectedSDG ? `Most common words/phrases for SDG ${selectedSDG.sdg}` : ''} + title={selectedSDG ? `Most common words/phrases for SDG ${selectedSDG.id}` : ''} open={selectedSDG !== null} >
From 179bc84a2b79e2105cee583e77f9e5391caeb3a8 Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 15:17:03 +0300 Subject: [PATCH 08/31] Feat(VNRAnalysis): added 'our approach' modal --- src/Priorities/VNRAnalysis.tsx | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/Priorities/VNRAnalysis.tsx b/src/Priorities/VNRAnalysis.tsx index 1f1faa0..4ed37c7 100644 --- a/src/Priorities/VNRAnalysis.tsx +++ b/src/Priorities/VNRAnalysis.tsx @@ -75,6 +75,8 @@ export const VNRAnalysis = (props: Props) => { const [hoveredSDG, setHoveredSDG] = useState(null); const [nodeData, setNodeData] = useState(null); const [showSalienceGraph, setShowSalienceGraph] = useState(false); + const [isApproachModalVisible, setIsApproachModalVisible] = useState(false); + const dataWithStatuses = data.map((d: any) => { const goalStatus = goalStatuses.find((el) => el.goal === d.id); return { @@ -114,8 +116,29 @@ export const VNRAnalysis = (props: Props) => { {document && document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} ) : document && document.map((d: any, i: number) => {d})} -
+
setIsApproachModalVisible(true)} style={{ cursor: 'pointer', color: 'var(--gray-600)', textDecoration: 'underline' }}> + Read about our approach +
+ setIsApproachModalVisible(false)} + onOk={() => setIsApproachModalVisible(false)} + > +
+ Countries national priorities are generated using machine learning to reveal the most prominent SDGs referenced in national policy documents. This analysis uses a custom-built model for SDG classification. + {' '} +
+
+ {' '} + The training data is based on an improved + {' '} + OSDG Community Dataset + . It considers 100k+ terms, including phrases and expressions. +
+
From a50b9b78d0cfc3ca6cd794fc93993befb85c3f55 Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 15:18:50 +0300 Subject: [PATCH 09/31] Refactor(index): refactored to use api calls to retrieve and select metadata --- src/Priorities/index.tsx | 418 ++++++++++++++++++++++----------------- 1 file changed, 232 insertions(+), 186 deletions(-) diff --git a/src/Priorities/index.tsx b/src/Priorities/index.tsx index 239529f..d11731e 100644 --- a/src/Priorities/index.tsx +++ b/src/Priorities/index.tsx @@ -1,19 +1,19 @@ +/* eslint-disable no-underscore-dangle */ /* eslint-disable jsx-a11y/label-has-associated-control */ import styled from 'styled-components'; -import axios, { AxiosResponse } from 'axios'; import { + Collapse, Modal, Radio, Select, Tabs, } from 'antd'; import { useEffect, useRef, useState } from 'react'; -import sortBy from 'lodash.sortby'; -import reverse from 'lodash.reverse'; -import { json } from 'd3-request'; +import { ChevronDown, ChevronUp } from 'lucide-react'; import { VNRAnalysis } from './VNRAnalysis'; import { GoalStatusType } from '../Types'; -import { API_ACCESS_TOKEN, DATASOURCELINK } from '../Constants'; import IMAGES from '../img/images'; +import { fetchDocumentById, fetchMetadata, submitDocumentsForAnalysis } from '../api/prioritiesCall'; +import { extractTextFromMultiplePdfs } from '../utils/extractPDF'; interface Props { countrySelected: string; @@ -66,6 +66,8 @@ const FileAttachmentButton = styled.input` display: none; `; +const FILES_LIMIT = 20; + export const Priorities = (props: Props) => { const { countrySelected, @@ -85,6 +87,41 @@ export const Priorities = (props: Props) => { const [data, setData] = useState(null); const [countryVNRs, setCountryVNRs] = useState(null); const [strategy, setStrategy] = useState<'equal' | 'proportional'>('equal'); + const [selectedDocument, setSelectedDocument] = useState(); + + useEffect(() => { + const fetchData = async () => { + try { + const metadata = await fetchMetadata(countrySelected); + setCountryFilePresent(metadata.length > 0); + if (metadata.length > 0) { + setCountryVNRs(metadata); + setSelectedDocument(metadata[0]._id); + } else { + setError('No VNRs'); + } + } catch (err) { + setError(err); + } + }; + + fetchData(); + }, [countrySelected]); + + useEffect(() => { + const fetchDocument = async () => { + try { + if (selectedDocument) { + const documentData = await fetchDocumentById(selectedDocument); + setData({ mode: 'defaultDocs', data: documentData.sdgs, documents: documentData.metadata }); + } + } catch (err) { + setError(err); + } + }; + + fetchDocument(); + }, [selectedDocument]); const handleFileSelect = (event: any) => { if (event.target.files) { @@ -99,97 +136,74 @@ export const Priorities = (props: Props) => { } }; - const analyzeDocument = () => { - if (selectedFileNotAnalyzed) { - const formData = new FormData(); - for (let i = 0; i < selectedFileNotAnalyzed.length; i += 1) { - formData.append('files', selectedFileNotAnalyzed[i]); + const handleFileRemove = (fileToRemove: any) => { + const updatedFiles = selectedFileNotAnalyzed.filter((file: any) => file !== fileToRemove); + let totalSize = 0; + updatedFiles.forEach((file: any) => { totalSize += (file.size / 1024 / 1024); }); + setTotalFileSize(totalSize); + setNoOfFiles(updatedFiles.length); + setSelectedFileNotAnalyzed(updatedFiles); + }; + + const analyzeDocument = async () => { + try { + const textsFiles = await extractTextFromMultiplePdfs(selectedFileNotAnalyzed); + + const validFiles = textsFiles.filter((file) => !file.error && file.text); + const invalidFiles = textsFiles.filter((file) => file.error || !file.text); + + console.log('error files: ', invalidFiles); + + const plaintextFiles = validFiles.map((file) => { + const blob = new Blob([file.text], { type: 'text/plain' }); + return new File([blob], `${file.file_name}.txt`, { type: 'text/plain' }); + }); + + if (plaintextFiles.length === 0) { + setError('No valid text files to submit for analysis.'); + setLoading(false); + return; } - setSelectedFile(selectedFileNotAnalyzed); - axios({ - method: 'post', - url: `https://sdg-push-diagnostic-api.azurewebsites.net/v1/upload/files?strategy=${strategy}`, - data: formData, - headers: { - 'Content-Type': 'multipart/form-data', - access_token: API_ACCESS_TOKEN, - }, - }) - .then((response: AxiosResponse) => { - if (typeof response.data === 'string') setError('PDF File Required'); - else { - setData({ mode: 'analyze', data: response.data.sdgs }); - setLoading(false); - setError(null); - } - }) - .catch((err) => { - setError(err.message); - setLoading(false); - }); + + const pagesArray = validFiles.map((file) => Number(file.pageCount)); + const response = await submitDocumentsForAnalysis( + plaintextFiles, + strategy === 'proportional' ? pagesArray : undefined, + ); + + const filesWithoutTxtExtension = plaintextFiles.map((file) => { + const originalFileName = file.name.replace(/\.txt$/, ''); + return new File([file], originalFileName, { type: file.type }); + }); + + setSelectedFile(filesWithoutTxtExtension); + setData({ mode: 'analyze', data: response.sdgs }); + setLoading(false); + setError(null); + } catch (err) { + setError(err); + setLoading(false); } }; - const analyzeVNR = () => { - if (selectYear) { - const { language } = countryVNRs[countryVNRs.findIndex((d: any) => d.year === selectYear)]; - axios.get( - `https://sdg-push-diagnostic-api.azurewebsites.net/v1/vnrs/find?iso=${countrySelected.toLowerCase()}&year=${selectYear}&language=${language}`, - { - headers: { access_token: API_ACCESS_TOKEN }, - }, - ) - .then((res) => { - setData(res.data.sdgs); + const analyzeVNR = async () => { + try { + if (selectYear) { + const metadata = await fetchMetadata(countrySelected, selectYear); + if (metadata.length > 0) { + const documentData = await fetchDocumentById(metadata[0].id); + setData(documentData.sdgs); setVNRYear(selectYear); setError(null); - }) - .catch((errorFetchingVNR) => { - setError(errorFetchingVNR.message); - }); + } else { + setError('No VNRs'); + } + } + } catch (err) { + setError(err); } }; - useEffect(() => { - setCountryFilePresent(undefined); - setData(undefined); - json(`${DATASOURCELINK}/PrioritiesData/${countrySelected}.json`, (err: any, d: any) => { - if (err) { - setCountryFilePresent(false); - axios.get( - 'https://sdg-push-diagnostic-api.azurewebsites.net/v1/vnrs/list', - { - headers: { access_token: API_ACCESS_TOKEN }, - }, - ) - .then((response:AxiosResponse) => { - const countryData = reverse(sortBy(response.data.filter((country: any) => country.iso === countrySelected.toLowerCase()), 'year')); - setCountryVNRs(countryData); - if (countryData.length > 0) { - setVNRYear(countryData[0].year); - setSelectYear(countryData[0].year); - axios.get( - `https://sdg-push-diagnostic-api.azurewebsites.net/v1/vnrs/find?iso=${countrySelected.toLowerCase()}&year=${countryData[0].year}&language=${countryData[0].language}`, - { - headers: { access_token: API_ACCESS_TOKEN }, - }, - ) - .then((res) => { - setData(res.data.sdgs); - }) - .catch((errorFetchingVNR) => { - setError(errorFetchingVNR.message); - }); - } else { - setError('No VNRs'); - } - }); - } else { - setCountryFilePresent(true); - setData({ mode: 'defaultDocs', data: d.sdgs, documents: d.doc_name }); - } - }); - }, [countrySelected]); return ( <> @@ -241,13 +255,13 @@ export const Priorities = (props: Props) => {
Countries national priorities are generated using machine learning to reveal the most prominent SDGs referenced in national policy documents. This analysis uses a custom-built model for SDG classification. The training data is based on an improved {' '} - OSDG Community Dataset + OSDG Community Dataset . It considers 100k+ terms, including phrases and expressions.

Documents such as {' '} - Voluntary National Reviews (VNRs) + Voluntary National Reviews (VNRs) {' '} indicates priorities of the government that can be mapped to the SDGs. These priorities are important as we develop the SDG Push interventions by country.
@@ -255,68 +269,82 @@ export const Priorities = (props: Props) => { Explore the analysis of these priorities using Machine Learning.
{ - countryVNRs - ? countryVNRs.length > 0 - ? ( - <> -
-

Select year

- -
- - - ) - : ( - <> -
- { setSelectYear(value); }} + > + { + countryVNRs.map((d: any, i: number) => ( + + {d.year} + + )) + } + +
+ + + ) + : ( + <> +
+ { setSelectedDocument(value); }} + > + { + countryVNRs.map((d: any, i: number) => ( + + {d.name} + + )) + } + + ), }, { @@ -330,7 +358,13 @@ export const Priorities = (props: Props) => {
Explore the analysis of these priorities using Machine Learning. {' '} - Maximum 10 documents allowed + + Upto + {' '} + {FILES_LIMIT} + {' '} + documents allowed + <>
@@ -347,7 +381,7 @@ export const Priorities = (props: Props) => { {(d.size / 1024 / 1024).toFixed(1)} {' '} MBs) - { setSelectedFileNotAnalyzed(selectedFileNotAnalyzed.filter((el: any) => d !== el)); }} /> + handleFileRemove(d)} />
)) } @@ -362,44 +396,56 @@ export const Priorities = (props: Props) => {
-

Document Weighting Strategy

- { setStrategy(target.target.value); }}> - Place Equal Weight on All Documents - Place More Weight on Longer Documents - + (isActive ? () : ())} + size='small' + expandIconPosition='end' + > + +

Document Weighting Strategy

+ { setStrategy(target.target.value); }}> + Place Equal Weight on All Documents +
+ Place More Weight on Longer Documents + + +
{ - selectedFileNotAnalyzed.length > 0 && noOfFiles < 11 && totalFileSize <= 100 - ? ( - - ) - : ( - - ) - } + selectedFileNotAnalyzed.length > 0 && noOfFiles <= FILES_LIMIT && totalFileSize <= 150 + ? ( + + ) + : ( + + ) + } { - noOfFiles > 10 || totalFileSize > 100 ? ( -

- { - noOfFiles > 10 ? `Maximum 10 files allowed (please remove ${noOfFiles - 10} files). ` : '' - } - { - totalFileSize > 100 ? 'Maximum total file size allowed is 100Mbs' : '' - } -

- ) : null - } + noOfFiles > FILES_LIMIT || totalFileSize > 150 ? ( +

+ { + noOfFiles > FILES_LIMIT ? `You have exceeded the limit of ${FILES_LIMIT} files. Please remove ${noOfFiles - FILES_LIMIT} file(s)). ` : '' + } + { + totalFileSize > 150 ? 'Maximum total file size allowed is 200Mbs' : '' + } +

+ ) : null + } ), @@ -440,7 +486,7 @@ export const Priorities = (props: Props) => { d.name) : data.mode === 'defaultDocs' ? data.documents : [`VNR ${vnrYear}`]} + document={data.mode === 'analyze' ? selectedFile.map((d: any) => d.name) : data.mode === 'defaultDocs' ? data.documents : [`VNR ${vnrYear}`]} defaultDocs={data.mode === 'defaultDocs'} onlyBubbleChart={false} /> From ab7852a5be7ccd0b039ccff2c7285c39f3f36977 Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 17:49:45 +0300 Subject: [PATCH 10/31] Feat(constants): added the required endpoints --- src/Constants.ts | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/Constants.ts b/src/Constants.ts index a290075..6de9265 100644 --- a/src/Constants.ts +++ b/src/Constants.ts @@ -32,12 +32,14 @@ export const FUTURESCENARIOINDICATORS = [ { Goal: 'SDG 1', Indicator: 'Female poverty head count (number of people)', - RelatedIndicator: 'Sex difference poverty headcount, age 25-34 (females minus males, number of people)', + RelatedIndicator: + 'Sex difference poverty headcount, age 25-34 (females minus males, number of people)', ID: 'indicator_1_3', }, { Goal: 'SDG 1', - Indicator: 'Sex difference poverty headcount, age 25-34 (females minus males, number of people)', + Indicator: + 'Sex difference poverty headcount, age 25-34 (females minus males, number of people)', RelatedIndicator: 'Female poverty head count (number of people)', ID: 'indicator_1_4', }, @@ -115,7 +117,13 @@ export const FUTURESCENARIOINDICATORS = [ }, ]; -export const COLORSCALE = ['#ffeda0', '#feb24c', '#fc4e2a', '#bd0026', '#800026']; +export const COLORSCALE = [ + '#ffeda0', + '#feb24c', + '#fc4e2a', + '#bd0026', + '#800026', +]; export const SDGGOALS = [ 'SDG 1: No Poverty', @@ -519,8 +527,6 @@ export const FIVE_P = [ }, ]; -export const API_ACCESS_TOKEN = 'ph92BXomD13M0qS9'; - export const REGION_FULL_NAME = [ { id: 'Global', @@ -656,3 +662,19 @@ export const COUNTRIES_WITH_REPORT = [ 'TLS', 'THA', ]; + +export const PDFJS_DIST_CDNS = [ + '//cdn.jsdelivr.net/npm/pdfjs-dist@4.6.82/build/pdf.worker.min.mjs', + '//cdnjs.cloudflare.com/ajax/libs/pdf.js/4.6.82/pdf.worker.min.mjs', + '//unpkg.com/browse/pdfjs-dist@4.6.82/build/pdf.worker.min.mjs', + '//cdn.jsdelivr.net/npm/pdfjs-dist@4.6.82/legacy/build/pdf.worker.min.mjs', +]; + +export const DIAGNOSTICS_API_BASE_URL = import.meta.env.VITE_DIAGNOSTICS_API_BASE_URL; +export const DIAGNOSTICS_API_ACCESS_TOKEN = import.meta.env.VITE_DIAGNOSTICS_API_ACCESS_TOKEN; + +export const TEXT_EXTRACTION_API_BASE_URL = import.meta.env.VITE_TEXT_EXTRACTION_API_BASE_URL; +export const TEXT_EXTRACTION_API_KEY = import.meta.env.VITE_TEXT_EXTRACTION_API_KEY; + +export const LANGUAGE_DETECTION_API_BASE_URL = import.meta.env.VITE_LANGUAGE_DETECTION_API_BASE_URL; +export const LANGUAGE_DETECTION_API_KEY = import.meta.env.VITE_LANGUAGE_DETECTION_API_KEY; From 1d7e9c15f971075da9e389e017354eec65b375c5 Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 17:50:16 +0300 Subject: [PATCH 11/31] Feat(env): added configuration required to import meta env --- src/vite-env.d.ts | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src/vite-env.d.ts diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts new file mode 100644 index 0000000..a08026d --- /dev/null +++ b/src/vite-env.d.ts @@ -0,0 +1,2 @@ +/// +/// From 934be58f3c48edda276dce604361052500ad2439 Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 17:51:14 +0300 Subject: [PATCH 12/31] Feat(priorities): added state to store invalid documents from extraction passed to VNRAnalysis --- src/Priorities/index.tsx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Priorities/index.tsx b/src/Priorities/index.tsx index d11731e..e98461d 100644 --- a/src/Priorities/index.tsx +++ b/src/Priorities/index.tsx @@ -21,6 +21,14 @@ interface Props { goalStatuses: GoalStatusType[]; } +type InvalidFileDetail = { + // eslint-disable-next-line camelcase + file_name: string; + text: string | number | any; + pageCount: string | number; + error: boolean; +}; + const HeroImageEl = styled.div` background: url(${IMAGES.heroImage}) rgba(0, 0, 0, 0.3) no-repeat center; background-size: cover; @@ -88,6 +96,7 @@ export const Priorities = (props: Props) => { const [countryVNRs, setCountryVNRs] = useState(null); const [strategy, setStrategy] = useState<'equal' | 'proportional'>('equal'); const [selectedDocument, setSelectedDocument] = useState(); + const [invalidFilesDetail, setInvalidFilesDetail] = useState(undefined); useEffect(() => { const fetchData = async () => { @@ -152,7 +161,7 @@ export const Priorities = (props: Props) => { const validFiles = textsFiles.filter((file) => !file.error && file.text); const invalidFiles = textsFiles.filter((file) => file.error || !file.text); - console.log('error files: ', invalidFiles); + setInvalidFilesDetail(invalidFiles); const plaintextFiles = validFiles.map((file) => { const blob = new Blob([file.text], { type: 'text/plain' }); @@ -489,6 +498,7 @@ export const Priorities = (props: Props) => { document={data.mode === 'analyze' ? selectedFile.map((d: any) => d.name) : data.mode === 'defaultDocs' ? data.documents : [`VNR ${vnrYear}`]} defaultDocs={data.mode === 'defaultDocs'} onlyBubbleChart={false} + invalidDocuments={invalidFilesDetail} /> ) : null From ffadcf01801536516316b521c3719a2a2609425b Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 17:58:51 +0300 Subject: [PATCH 13/31] Refactor(extractPDF): updated error message to be concise --- src/utils/extractPDF.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/utils/extractPDF.ts b/src/utils/extractPDF.ts index 9bd2816..a9aafc3 100644 --- a/src/utils/extractPDF.ts +++ b/src/utils/extractPDF.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop */ import axios from 'axios'; import { GlobalWorkerOptions, @@ -65,7 +66,7 @@ async function detectLanguage(text: string) { return response.data.official; } catch (err: any) { - throw new Error(`Language detection API failed: ${err.message}`); + throw new Error('Language detection failed for this document'); } } @@ -90,7 +91,7 @@ async function extractViaAPI(fileData: File) { response.data.pages.length, ]; } catch (err: any) { - throw new Error(`Text extraction failed ${err}`); + throw new Error('Text extraction failed for this document'); } } @@ -107,9 +108,7 @@ async function pdfToText(file: File) { pageCount = numPages; for (let pageNumber = 1; pageNumber <= numPages; pageNumber += 1) { - // eslint-disable-next-line no-await-in-loop const page: PDFPageProxy = await pdf.getPage(pageNumber); - // eslint-disable-next-line no-await-in-loop const textContent = await page.getTextContent(); const pageText = textContent.items .map((item) => ('str' in item ? item.str : '')) @@ -137,7 +136,7 @@ async function pdfToText(file: File) { const language = await detectLanguage(extractedText.trim()); if (language !== 'en') { - throw new Error(`Non-English document detected: ${file.name}`); + throw new Error('Document contains Non-English content'); } return [extractedText.trim(), pageCount]; From abae7851ac06c642c371635f48590087266b79b7 Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 18:07:01 +0300 Subject: [PATCH 14/31] Feat(VNRAnalysis): added invalidDocument props and uses popover to display 'analysis based on' --- src/Priorities/VNRAnalysis.tsx | 85 ++++++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 13 deletions(-) diff --git a/src/Priorities/VNRAnalysis.tsx b/src/Priorities/VNRAnalysis.tsx index 4ed37c7..234d509 100644 --- a/src/Priorities/VNRAnalysis.tsx +++ b/src/Priorities/VNRAnalysis.tsx @@ -1,7 +1,7 @@ /* eslint-disable jsx-a11y/no-static-element-interactions */ /* eslint-disable jsx-a11y/click-events-have-key-events */ import { useEffect, useState } from 'react'; -import { Modal } from 'antd'; +import { Button, Modal, Popover } from 'antd'; import styled from 'styled-components'; import { forceCollide, forceManyBody, forceSimulation, forceX, forceY, @@ -19,6 +19,7 @@ interface Props { document: any; defaultDocs: boolean; onlyBubbleChart: boolean; + invalidDocuments: any; } interface SDGHoveredProps { @@ -63,6 +64,13 @@ const FileNameChip = styled.div` font-weight: bold; `; +const FileNameErrorChip = styled.div` +font-size: 1rem; + padding: 0.5rem; + background-color: var(--light-red); + font-weight: bold; +`; + export const VNRAnalysis = (props: Props) => { const { data, @@ -70,6 +78,7 @@ export const VNRAnalysis = (props: Props) => { goalStatuses, defaultDocs, onlyBubbleChart, + invalidDocuments, } = props; const [selectedSDG, setSelectedSDG] = useState(null); const [hoveredSDG, setHoveredSDG] = useState(null); @@ -111,15 +120,40 @@ export const VNRAnalysis = (props: Props) => {

National Priorities Based on

- {defaultDocs ? ( - <> - {document && document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} - - ) : document && document.map((d: any, i: number) => {d})} + + {document && document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} + + ) + : ( + <> + {document && document.map((d: any, i: number) => {d})} +
+ {invalidDocuments && invalidDocuments.length > 0 && Excluded files} + {invalidDocuments.map((d: any, i: number) => ( + + + {' '} + {d.file_name} + + + ))} +
+ + )} + > + +
-
setIsApproachModalVisible(true)} style={{ cursor: 'pointer', color: 'var(--gray-600)', textDecoration: 'underline' }}> + setIsApproachModalVisible(true)} style={{ cursor: 'pointer', color: 'var(--gray-600)', textDecoration: 'underline' }}> Read about our approach -
+ {

Comparing SDG trends and SDG national priorities based on

- {defaultDocs ? ( - <> - {document && document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} - - ) : document && document.map((d: any, i: number) => {d})} + + {document && document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} + + ) + : ( + <> + {document && document.map((d: any, i: number) => {d})} +
+ {invalidDocuments && invalidDocuments.length > 0 && Excluded files} + {invalidDocuments.map((d: any, i: number) => ( + + + {' '} + {d.file_name} + + + ))} +
+ + )} + > + +

This matrix maps the SDGs along two parameters From 0b96ce37879fb42d0ad968843aca4443e22a6369 Mon Sep 17 00:00:00 2001 From: GedionT Date: Wed, 25 Sep 2024 20:26:20 +0300 Subject: [PATCH 15/31] Fix(detailedReportView): added a required prop to VNRAnalysis --- src/HomePage/DetailedReportView.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/HomePage/DetailedReportView.tsx b/src/HomePage/DetailedReportView.tsx index 787323d..dca76c5 100644 --- a/src/HomePage/DetailedReportView.tsx +++ b/src/HomePage/DetailedReportView.tsx @@ -251,6 +251,7 @@ export const DetailedReportView = (props: Props) => { document={priorityData.documents} defaultDocs onlyBubbleChart + invalidDocuments={undefined} />

{ }} > { - interlinkage.LinkageType[0] === 'synergies' ? 'Synergies' : interlinkage.LinkageType[0] === 'tradeOffs' ? 'Trade-Offs' : 'Not Specified' - } + interlinkage.LinkageType[0] === 'synergies' ? 'Synergies' : interlinkage.LinkageType[0] === 'tradeOffs' ? 'Trade-Offs' : 'Not Specified' + }
Date: Thu, 26 Sep 2024 16:00:20 +0300 Subject: [PATCH 16/31] Fix(ci/cd): added env import in yaml file --- .../azure-static-web-apps-gentle-island-066846403.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/azure-static-web-apps-gentle-island-066846403.yml b/.github/workflows/azure-static-web-apps-gentle-island-066846403.yml index e08e4ab..a325875 100644 --- a/.github/workflows/azure-static-web-apps-gentle-island-066846403.yml +++ b/.github/workflows/azure-static-web-apps-gentle-island-066846403.yml @@ -14,6 +14,13 @@ jobs: if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.action != 'closed') runs-on: ubuntu-latest name: Build and Deploy Job + env: + VITE_DIAGNOSTICS_API_BASE_URL: ${{secrets.VITE_DIAGNOSTICS_API_BASE_URL}} + VITE_DIAGNOSTICS_API_ACCESS_TOKEN: ${{secrets.VITE_DIAGNOSTICS_API_ACCESS_TOKEN}} + VITE_LANGUAGE_DETECTION_API_BASE_URL: ${{secrets.VITE_LANGUAGE_DETECTION_API_BASE_URL}} + VITE_LANGUAGE_DETECTION_API_KEY: ${{secrets.VITE_LANGUAGE_DETECTION_API_KEY}} + VITE_TEXT_EXTRACTION_API_BASE_URL: ${{secrets.VITE_TEXT_EXTRACTION_API_BASE_URL}} + VITE_TEXT_EXTRACTION_API_KEY: ${{secrets.VITE_TEXT_EXTRACTION_API_KEY}} steps: - uses: actions/checkout@v3 with: From a3569a04c65191a2d94a7f4c5efc8903452a2306 Mon Sep 17 00:00:00 2001 From: GedionT Date: Fri, 27 Sep 2024 16:21:56 +0300 Subject: [PATCH 17/31] Fix(VNRAnalysis): added margin to FileNameChip tag style --- src/Priorities/VNRAnalysis.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Priorities/VNRAnalysis.tsx b/src/Priorities/VNRAnalysis.tsx index 234d509..d036a26 100644 --- a/src/Priorities/VNRAnalysis.tsx +++ b/src/Priorities/VNRAnalysis.tsx @@ -62,6 +62,7 @@ const FileNameChip = styled.div` padding: 0.5rem; background-color: var(--gray-300); font-weight: bold; + margin: 2px; `; const FileNameErrorChip = styled.div` @@ -69,6 +70,7 @@ font-size: 1rem; padding: 0.5rem; background-color: var(--light-red); font-weight: bold; + margin: 2px; `; export const VNRAnalysis = (props: Props) => { From ebcd96da9ec8c145f7bc44fb80d478bb4ff52cfb Mon Sep 17 00:00:00 2001 From: GedionT Date: Fri, 27 Sep 2024 16:23:18 +0300 Subject: [PATCH 18/31] Feat(priorities): added status in modal while uploading document for analysis --- src/Priorities/index.tsx | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/Priorities/index.tsx b/src/Priorities/index.tsx index e98461d..46a5b6b 100644 --- a/src/Priorities/index.tsx +++ b/src/Priorities/index.tsx @@ -95,6 +95,7 @@ export const Priorities = (props: Props) => { const [data, setData] = useState(null); const [countryVNRs, setCountryVNRs] = useState(null); const [strategy, setStrategy] = useState<'equal' | 'proportional'>('equal'); + const [status, setStatus] = useState(null); const [selectedDocument, setSelectedDocument] = useState(); const [invalidFilesDetail, setInvalidFilesDetail] = useState(undefined); @@ -156,8 +157,11 @@ export const Priorities = (props: Props) => { const analyzeDocument = async () => { try { + setStatus(`Extracting texts and identifying language from ${selectedFileNotAnalyzed.length} documents...`); const textsFiles = await extractTextFromMultiplePdfs(selectedFileNotAnalyzed); + setStatus('Filtering documents...'); + const validFiles = textsFiles.filter((file) => !file.error && file.text); const invalidFiles = textsFiles.filter((file) => file.error || !file.text); @@ -174,6 +178,7 @@ export const Priorities = (props: Props) => { return; } + setStatus('Analyzing documents...'); const pagesArray = validFiles.map((file) => Number(file.pageCount)); const response = await submitDocumentsForAnalysis( plaintextFiles, @@ -185,9 +190,12 @@ export const Priorities = (props: Props) => { return new File([file], originalFileName, { type: file.type }); }); + setStatus('Analysis complete'); + setSelectedFile(filesWithoutTxtExtension); setData({ mode: 'analyze', data: response.sdgs }); setLoading(false); + setStatus(null); setError(null); } catch (err) { setError(err); @@ -262,12 +270,12 @@ export const Priorities = (props: Props) => { children: !data && countryFilePresent === false ? ( <>
- Countries national priorities are generated using machine learning to reveal the most prominent SDGs referenced in national policy documents. This analysis uses a custom-built model for SDG classification. The training data is based on an improved + {/* Countries national priorities are generated using machine learning to reveal the most prominent SDGs referenced in national policy documents. This analysis uses a custom-built model for SDG classification. The training data is based on an improved {' '} OSDG Community Dataset . It considers 100k+ terms, including phrases and expressions.
-
+
*/} Documents such as {' '} Voluntary National Reviews (VNRs) @@ -511,6 +519,14 @@ export const Priorities = (props: Props) => {
+
+

+ {status} +

+
); From bf00accdbe1ab2b14fe10d43c66f2fd773cfcadb Mon Sep 17 00:00:00 2001 From: GedionT Date: Fri, 27 Sep 2024 16:23:46 +0300 Subject: [PATCH 19/31] Refactor(extractPDF): error message refactor for more clarity --- src/utils/extractPDF.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/utils/extractPDF.ts b/src/utils/extractPDF.ts index a9aafc3..8fa4b4e 100644 --- a/src/utils/extractPDF.ts +++ b/src/utils/extractPDF.ts @@ -134,6 +134,10 @@ async function pdfToText(file: File) { } } + if (extractedText.trim().length === 0) { + throw new Error('Unable to extract text from this document'); + } + const language = await detectLanguage(extractedText.trim()); if (language !== 'en') { throw new Error('Document contains Non-English content'); From bf7dd2989922e0a23dc24a5bdde04226339cc045 Mon Sep 17 00:00:00 2001 From: GedionT Date: Thu, 3 Oct 2024 20:57:02 +0300 Subject: [PATCH 20/31] Refactor(VNRAnalysis): moved methodology modal to Priorities/index tabs --- src/Priorities/VNRAnalysis.tsx | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/Priorities/VNRAnalysis.tsx b/src/Priorities/VNRAnalysis.tsx index d036a26..b4b2192 100644 --- a/src/Priorities/VNRAnalysis.tsx +++ b/src/Priorities/VNRAnalysis.tsx @@ -1,5 +1,3 @@ -/* eslint-disable jsx-a11y/no-static-element-interactions */ -/* eslint-disable jsx-a11y/click-events-have-key-events */ import { useEffect, useState } from 'react'; import { Button, Modal, Popover } from 'antd'; import styled from 'styled-components'; @@ -86,7 +84,6 @@ export const VNRAnalysis = (props: Props) => { const [hoveredSDG, setHoveredSDG] = useState(null); const [nodeData, setNodeData] = useState(null); const [showSalienceGraph, setShowSalienceGraph] = useState(false); - const [isApproachModalVisible, setIsApproachModalVisible] = useState(false); const dataWithStatuses = data.map((d: any) => { const goalStatus = goalStatuses.find((el) => el.goal === d.id); @@ -153,28 +150,6 @@ export const VNRAnalysis = (props: Props) => {
- setIsApproachModalVisible(true)} style={{ cursor: 'pointer', color: 'var(--gray-600)', textDecoration: 'underline' }}> - Read about our approach - - setIsApproachModalVisible(false)} - onOk={() => setIsApproachModalVisible(false)} - > -
- Countries national priorities are generated using machine learning to reveal the most prominent SDGs referenced in national policy documents. This analysis uses a custom-built model for SDG classification. - {' '} -
-
- {' '} - The training data is based on an improved - {' '} - OSDG Community Dataset - . It considers 100k+ terms, including phrases and expressions. -
-
From f29ebd839543baa8984c2a87c50fd75aea43ac7b Mon Sep 17 00:00:00 2001 From: GedionT Date: Thu, 3 Oct 2024 21:55:25 +0300 Subject: [PATCH 21/31] Refactor(constants): unified api into one endpoint --- src/Constants.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Constants.ts b/src/Constants.ts index 6de9265..97f888a 100644 --- a/src/Constants.ts +++ b/src/Constants.ts @@ -670,11 +670,5 @@ export const PDFJS_DIST_CDNS = [ '//cdn.jsdelivr.net/npm/pdfjs-dist@4.6.82/legacy/build/pdf.worker.min.mjs', ]; -export const DIAGNOSTICS_API_BASE_URL = import.meta.env.VITE_DIAGNOSTICS_API_BASE_URL; -export const DIAGNOSTICS_API_ACCESS_TOKEN = import.meta.env.VITE_DIAGNOSTICS_API_ACCESS_TOKEN; - -export const TEXT_EXTRACTION_API_BASE_URL = import.meta.env.VITE_TEXT_EXTRACTION_API_BASE_URL; -export const TEXT_EXTRACTION_API_KEY = import.meta.env.VITE_TEXT_EXTRACTION_API_KEY; - -export const LANGUAGE_DETECTION_API_BASE_URL = import.meta.env.VITE_LANGUAGE_DETECTION_API_BASE_URL; -export const LANGUAGE_DETECTION_API_KEY = import.meta.env.VITE_LANGUAGE_DETECTION_API_KEY; +export const AIAAS_API_BASE_URL = import.meta.env.VITE_AIAAS_API_BASE_URL; +export const AIAAS_API_KEY = import.meta.env.VITE_AIAAS_API_KEY; From cecc905a2011495b2e0a15996e2ee4279e0f9d5e Mon Sep 17 00:00:00 2001 From: GedionT Date: Thu, 3 Oct 2024 21:56:21 +0300 Subject: [PATCH 22/31] Refactor(prioritiesCall): placed all aiaas api calls in one module --- src/api/prioritiesCall.ts | 84 +++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/src/api/prioritiesCall.ts b/src/api/prioritiesCall.ts index 0812e1b..cb06ce5 100644 --- a/src/api/prioritiesCall.ts +++ b/src/api/prioritiesCall.ts @@ -1,8 +1,5 @@ import axios from 'axios'; -import { - DIAGNOSTICS_API_BASE_URL, - DIAGNOSTICS_API_ACCESS_TOKEN, -} from '../Constants'; +import { AIAAS_API_BASE_URL, AIAAS_API_KEY } from '../Constants'; export const fetchMetadata = async ( iso: string, @@ -16,29 +13,29 @@ export const fetchMetadata = async ( language, kind, }; - const response = await axios.get(`${DIAGNOSTICS_API_BASE_URL}/metadata`, { - params, - headers: { - accept: 'application/json', - api_key: DIAGNOSTICS_API_ACCESS_TOKEN, - }, - }); - return response.data; -}; - -export const fetchDocumentById = async (id: string) => { const response = await axios.get( - `${DIAGNOSTICS_API_BASE_URL}/documents/${id}`, + `${AIAAS_API_BASE_URL}/classification/metadata`, { + params, headers: { accept: 'application/json', - api_key: DIAGNOSTICS_API_ACCESS_TOKEN, + api_key: AIAAS_API_KEY, }, }, ); return response.data; }; +export const fetchDocumentById = async (id: string) => { + const response = await axios.get(`${AIAAS_API_BASE_URL}/classification/diagnostics/${id}`, { + headers: { + accept: 'application/json', + api_key: AIAAS_API_KEY, + }, + }); + return response.data; +}; + export const submitDocumentsForAnalysis = async ( files: File[], weights?: number[], @@ -47,20 +44,63 @@ export const submitDocumentsForAnalysis = async ( files.forEach((file) => formData.append('files', file)); - const weightParam = weights && weights.length > 0 - ? `?${weights.map((weight) => `weights=${weight}`).join('&')}` - : ''; + const weightAndFeaturesParam = weights && weights.length > 0 + ? `?${weights.map((weight) => `weights=${weight}`).join('&')}&features=true` + : '?features=true'; const response = await axios.post( - `${DIAGNOSTICS_API_BASE_URL}/documents${weightParam}`, + `${AIAAS_API_BASE_URL}/classification/diagnostics${weightAndFeaturesParam}`, formData, { headers: { 'Content-Type': 'multipart/form-data', - api_key: DIAGNOSTICS_API_ACCESS_TOKEN, + api_key: AIAAS_API_KEY, }, }, ); return response.data; }; + +export async function detectLanguages(texts: string[]) { + try { + const response = await axios({ + method: 'post', + url: `${AIAAS_API_BASE_URL}/identification/languages`, + data: JSON.stringify({ texts }), + headers: { + 'Content-Type': 'application/json', + api_key: AIAAS_API_KEY, + }, + }); + + return response.data.map((result: any) => result[0]); + } catch (err: any) { + throw new Error('Language detection failed for the documents'); + } +} + +export async function extractViaAPI(fileData: File) { + const formData = new FormData(); + formData.append('file', fileData); + + try { + const response = await axios({ + method: 'post', + url: `${AIAAS_API_BASE_URL}/extraction`, + data: formData, + headers: { + 'Content-Type': 'multipart/form-data', + accept: 'application/json', + api_key: AIAAS_API_KEY, + }, + }); + + return [ + response.data.pages.map((page: any) => page.text).join(' '), + response.data.pages.length, + ]; + } catch (err: any) { + throw new Error('Text extraction failed for this document'); + } +} From 47334ae1ea58e00891e76e36ac58cdb8a82963c6 Mon Sep 17 00:00:00 2001 From: GedionT Date: Thu, 3 Oct 2024 21:57:44 +0300 Subject: [PATCH 23/31] Refactor(extractPDF): modularized api calls and batched language identification instead of individual identification --- src/utils/extractPDF.ts | 79 ++++++++++------------------------------- 1 file changed, 19 insertions(+), 60 deletions(-) diff --git a/src/utils/extractPDF.ts b/src/utils/extractPDF.ts index 8fa4b4e..c30e9f7 100644 --- a/src/utils/extractPDF.ts +++ b/src/utils/extractPDF.ts @@ -1,19 +1,12 @@ /* eslint-disable no-await-in-loop */ -import axios from 'axios'; import { GlobalWorkerOptions, getDocument, PDFDocumentProxy, PDFPageProxy, } from 'pdfjs-dist'; - -import { - LANGUAGE_DETECTION_API_BASE_URL, - LANGUAGE_DETECTION_API_KEY, - PDFJS_DIST_CDNS, - TEXT_EXTRACTION_API_BASE_URL, - TEXT_EXTRACTION_API_KEY, -} from '../Constants'; +import { detectLanguages, extractViaAPI } from '../api/prioritiesCall'; +import { PDFJS_DIST_CDNS } from '../Constants'; GlobalWorkerOptions.workerSrc = ''; @@ -24,12 +17,12 @@ async function checkCdn(cdn: string, index: number) { return cdn; } console.error( - `CDN ${index + 1} down or invalid MIME type: ${response.headers.get( + `CDN ${index + 1} down or invalid MIME type of : ${response.headers.get( 'Content-Type', )}`, ); } catch (error) { - console.error(`Error checking CDN ${index + 1}:`, error); + console.error(`CDN ${index + 1} down: `, error); } return null; } @@ -52,49 +45,6 @@ async function loadPdfWorker() { } } -async function detectLanguage(text: string) { - try { - const response = await axios({ - method: 'post', - url: LANGUAGE_DETECTION_API_BASE_URL, - data: JSON.stringify({ text }), - headers: { - 'Content-Type': 'application/json', - api_key: LANGUAGE_DETECTION_API_KEY, - }, - }); - - return response.data.official; - } catch (err: any) { - throw new Error('Language detection failed for this document'); - } -} - -async function extractViaAPI(fileData: File) { - const formData = new FormData(); - formData.append('file', fileData); - - try { - const response = await axios({ - method: 'post', - url: TEXT_EXTRACTION_API_BASE_URL, - data: formData, - headers: { - 'Content-Type': 'multipart/form-data', - accept: 'application/json', - api_key: TEXT_EXTRACTION_API_KEY, - }, - }); - - return [ - response.data.pages.map((page: any) => page.text).join(' '), - response.data.pages.length, - ]; - } catch (err: any) { - throw new Error('Text extraction failed for this document'); - } -} - async function pdfToText(file: File) { const blobUrl = URL.createObjectURL(file); let extractedText = ''; @@ -138,11 +88,6 @@ async function pdfToText(file: File) { throw new Error('Unable to extract text from this document'); } - const language = await detectLanguage(extractedText.trim()); - if (language !== 'en') { - throw new Error('Document contains Non-English content'); - } - return [extractedText.trim(), pageCount]; } @@ -170,5 +115,19 @@ export async function extractTextFromMultiplePdfs(files: File[]) { }), ); - return texts; + const validTexts = texts.filter((t) => !t.error).map((t) => t.text); + const languages = await detectLanguages(validTexts); + + const textResults = texts.map((t, index) => { + if (!t.error && languages[index].official !== 'en') { + return { + ...t, + error: true, + text: `Document contains ${languages[index].name} content that is currently unsupported by the model.`, + }; + } + return t; + }); + + return textResults; } From 1223565e40077017ec7e4f7dcf5b038850909e11 Mon Sep 17 00:00:00 2001 From: GedionT Date: Thu, 3 Oct 2024 21:58:32 +0300 Subject: [PATCH 24/31] Refactor(VNRAnalysis): fileNameChip displayed for one document and button with Tooltip for multiple files --- src/Priorities/VNRAnalysis.tsx | 136 +++++++++++++++++++-------------- 1 file changed, 78 insertions(+), 58 deletions(-) diff --git a/src/Priorities/VNRAnalysis.tsx b/src/Priorities/VNRAnalysis.tsx index b4b2192..276a197 100644 --- a/src/Priorities/VNRAnalysis.tsx +++ b/src/Priorities/VNRAnalysis.tsx @@ -64,7 +64,7 @@ const FileNameChip = styled.div` `; const FileNameErrorChip = styled.div` -font-size: 1rem; + font-size: 1rem; padding: 0.5rem; background-color: var(--light-red); font-weight: bold; @@ -119,36 +119,46 @@ export const VNRAnalysis = (props: Props) => {

National Priorities Based on

- - {document && document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} - - ) - : ( + {defaultDocs && document.length === 1 ? ( + + {document[0].link ? ( + {document[0].name} + ) : ( + document[0].name + )} + + ) : ( + - {document && document.map((d: any, i: number) => {d})} -
- {invalidDocuments && invalidDocuments.length > 0 && Excluded files} - {invalidDocuments.map((d: any, i: number) => ( - - - {' '} - {d.file_name} - - - ))} -
+ {document && document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} - )} - > - - + ) + : ( + <> + {document && document.map((d: any, i: number) => {d})} +
+ {invalidDocuments && invalidDocuments.length > 0 && Excluded files} + {invalidDocuments.map((d: any, i: number) => ( + + + {' '} + {d.file_name} + + + ))} +
+ + )} + > + + + )} @@ -324,36 +334,46 @@ export const VNRAnalysis = (props: Props) => {

Comparing SDG trends and SDG national priorities based on

- - {document && document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} - - ) - : ( + {defaultDocs && document.length === 1 ? ( + + {document[0].link ? ( + {document[0].name} + ) : ( + document[0].name + )} + + ) : ( + - {document && document.map((d: any, i: number) => {d})} -
- {invalidDocuments && invalidDocuments.length > 0 && Excluded files} - {invalidDocuments.map((d: any, i: number) => ( - - - {' '} - {d.file_name} - - - ))} -
+ {document && document.map((d: any, i: number) => (d.link ? {d.name} : {d.name}))} - )} - > - - + ) + : ( + <> + {document && document.map((d: any, i: number) => {d})} +
+ {invalidDocuments && invalidDocuments.length > 0 && Excluded files} + {invalidDocuments.map((d: any, i: number) => ( + + + {' '} + {d.file_name} + + + ))} +
+ + )} + > + + + )}

This matrix maps the SDGs along two parameters @@ -420,7 +440,7 @@ export const VNRAnalysis = (props: Props) => { >

{ - selectedSDG.features.length > 0 + selectedSDG.features && selectedSDG.features.length > 0 ? selectedSDG.features.map((d: any, i: number) =>
{d}
) : 'No words/phrases available' } From 328f8c0e56c7265792d0a8cb5e5a01d3ecd144e3 Mon Sep 17 00:00:00 2001 From: GedionT Date: Thu, 3 Oct 2024 21:59:38 +0300 Subject: [PATCH 25/31] Refactor(Priorities): Methodology modal applied in tabs, Accordion font minimized --- src/Priorities/index.tsx | 89 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 81 insertions(+), 8 deletions(-) diff --git a/src/Priorities/index.tsx b/src/Priorities/index.tsx index 46a5b6b..a835be7 100644 --- a/src/Priorities/index.tsx +++ b/src/Priorities/index.tsx @@ -98,6 +98,7 @@ export const Priorities = (props: Props) => { const [status, setStatus] = useState(null); const [selectedDocument, setSelectedDocument] = useState(); const [invalidFilesDetail, setInvalidFilesDetail] = useState(undefined); + const [isApproachModalVisible, setIsApproachModalVisible] = useState(false); useEffect(() => { const fetchData = async () => { @@ -157,7 +158,7 @@ export const Priorities = (props: Props) => { const analyzeDocument = async () => { try { - setStatus(`Extracting texts and identifying language from ${selectedFileNotAnalyzed.length} documents...`); + setStatus(`Extracting texts and identifying language from ${selectedFileNotAnalyzed.length} document(s)...`); const textsFiles = await extractTextFromMultiplePdfs(selectedFileNotAnalyzed); setStatus('Filtering documents...'); @@ -270,7 +271,7 @@ export const Priorities = (props: Props) => { children: !data && countryFilePresent === false ? ( <>
- {/* Countries national priorities are generated using machine learning to reveal the most prominent SDGs referenced in national policy documents. This analysis uses a custom-built model for SDG classification. The training data is based on an improved + {/* Countries' national priorities are generated using machine learning to reveal the most prominent SDGs referenced in national policy documents. This analysis uses a custom-built model for SDG classification. The training data is based on an improved {' '} OSDG Community Dataset . It considers 100k+ terms, including phrases and expressions. @@ -341,12 +342,48 @@ export const Priorities = (props: Props) => { ) : ( <> -
+
Documents such as National Development Plans indicate priorities of the government that can be mapped to the SDGs. These priorities are important as we develop the SDG Push interventions by country. + {' '} +

Explore the analysis of these priorities using Machine Learning.
+ setIsApproachModalVisible(false)} + onOk={() => setIsApproachModalVisible(false)} + > +
+ Countries' national priorities are generated using machine learning to reveal the most prominent SDGs referenced in national policy documents. This analysis uses a custom-built model for SDG classification. + {' '} +
+
+ {' '} + The training data is based on an improved + {' '} + OSDG Community Dataset + . It considers 100k+ terms, including phrases and expressions. +
+