From 70706e868e10a01299c8ab1f65523bdb06fa1b3b Mon Sep 17 00:00:00 2001 From: jshemas Date: Mon, 22 Jul 2024 17:27:10 -0400 Subject: [PATCH 1/7] copy isUrl as is --- lib/isURL.ts | 296 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 lib/isURL.ts diff --git a/lib/isURL.ts b/lib/isURL.ts new file mode 100644 index 0000000..cba32fe --- /dev/null +++ b/lib/isURL.ts @@ -0,0 +1,296 @@ +// This is from https://github.com/validatorjs/validator.js version: 13.12.0 + +// https://github.com/validatorjs/validator.js/blob/master/src/lib/util/assertString.js +function assertString(input) { + const isString = typeof input === 'string' || input instanceof String; + + if (!isString) { + let invalidType = typeof input; + if (input === null) invalidType = 'null'; + else if (invalidType === 'object') invalidType = input.constructor.name; + + throw new TypeError(`Expected a string but received a ${invalidType}`); + } +} + +// https://github.com/validatorjs/validator.js/blob/master/src/lib/isFQDN.js +const default_fqdn_options = { + require_tld: true, + allow_underscores: false, + allow_trailing_dot: false, + allow_numeric_tld: false, + allow_wildcard: false, + ignore_max_length: false, +}; + +function isFQDN(str, options) { + assertString(str); + options = merge(options, default_fqdn_options); + + /* Remove the optional trailing dot before checking validity */ + if (options.allow_trailing_dot && str[str.length - 1] === '.') { + str = str.substring(0, str.length - 1); + } + + /* Remove the optional wildcard before checking validity */ + if (options.allow_wildcard === true && str.indexOf('*.') === 0) { + str = str.substring(2); + } + + const parts = str.split('.'); + const tld = parts[parts.length - 1]; + + if (options.require_tld) { + // disallow fqdns without tld + if (parts.length < 2) { + return false; + } + + if (!options.allow_numeric_tld && !/^([a-z\u00A1-\u00A8\u00AA-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]{2,}|xn[a-z0-9-]{2,})$/i.test(tld)) { + return false; + } + + // disallow spaces + if (/\s/.test(tld)) { + return false; + } + } + + // reject numeric TLDs + if (!options.allow_numeric_tld && /^\d+$/.test(tld)) { + return false; + } + + return parts.every((part) => { + if (part.length > 63 && !options.ignore_max_length) { + return false; + } + + if (!/^[a-z_\u00a1-\uffff0-9-]+$/i.test(part)) { + return false; + } + + // disallow full-width chars + if (/[\uff01-\uff5e]/.test(part)) { + return false; + } + + // disallow parts starting or ending with hyphen + if (/^-|-$/.test(part)) { + return false; + } + + if (!options.allow_underscores && /_/.test(part)) { + return false; + } + + return true; + }); +} + +// https://github.com/validatorjs/validator.js/blob/master/src/lib/isIP.js +const IPv4SegmentFormat = '(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])'; +const IPv4AddressFormat = `(${IPv4SegmentFormat}[.]){3}${IPv4SegmentFormat}`; +const IPv4AddressRegExp = new RegExp(`^${IPv4AddressFormat}$`); + +const IPv6SegmentFormat = '(?:[0-9a-fA-F]{1,4})'; +const IPv6AddressRegExp = new RegExp('^(' + + `(?:${IPv6SegmentFormat}:){7}(?:${IPv6SegmentFormat}|:)|` + + `(?:${IPv6SegmentFormat}:){6}(?:${IPv4AddressFormat}|:${IPv6SegmentFormat}|:)|` + + `(?:${IPv6SegmentFormat}:){5}(?::${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,2}|:)|` + + `(?:${IPv6SegmentFormat}:){4}(?:(:${IPv6SegmentFormat}){0,1}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,3}|:)|` + + `(?:${IPv6SegmentFormat}:){3}(?:(:${IPv6SegmentFormat}){0,2}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,4}|:)|` + + `(?:${IPv6SegmentFormat}:){2}(?:(:${IPv6SegmentFormat}){0,3}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,5}|:)|` + + `(?:${IPv6SegmentFormat}:){1}(?:(:${IPv6SegmentFormat}){0,4}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,6}|:)|` + + `(?::((?::${IPv6SegmentFormat}){0,5}:${IPv4AddressFormat}|(?::${IPv6SegmentFormat}){1,7}|:))` + + ')(%[0-9a-zA-Z-.:]{1,})?$'); + +function isIP(str, version = '') { + assertString(str); + version = String(version); + if (!version) { + return isIP(str, 4) || isIP(str, 6); + } + if (version === '4') { + return IPv4AddressRegExp.test(str); + } + if (version === '6') { + return IPv6AddressRegExp.test(str); + } + return false; +} + +// https://github.com/validatorjs/validator.js/blob/master/src/lib/util/merge.js +function merge(obj = { }, defaults) { + for (const key in defaults) { + if (typeof obj[key] === 'undefined') { + obj[key] = defaults[key]; + } + } + return obj; +} + +// https://github.com/validatorjs/validator.js/blob/master/src/lib/isURL.js +/* +options for isURL method + +require_protocol - if set as true isURL will return false if protocol is not present in the URL +require_valid_protocol - isURL will check if the URL's protocol is present in the protocols option +protocols - valid protocols can be modified with this option +require_host - if set as false isURL will not check if host is present in the URL +require_port - if set as true isURL will check if port is present in the URL +allow_protocol_relative_urls - if set as true protocol relative URLs will be allowed +validate_length - if set as false isURL will skip string length validation (IE maximum is 2083) + +*/ +const default_url_options = { + protocols: ['http', 'https', 'ftp'], + require_tld: true, + require_protocol: false, + require_host: true, + require_port: false, + require_valid_protocol: true, + allow_underscores: false, + allow_trailing_dot: false, + allow_protocol_relative_urls: false, + allow_fragments: true, + allow_query_components: true, + validate_length: true, +}; + +const wrapped_ipv6 = /^\[([^\]]+)\](?::([0-9]+))?$/; + +function isRegExp(obj) { + return Object.prototype.toString.call(obj) === '[object RegExp]'; +} + +function checkHost(host, matches) { + for (let i = 0; i < matches.length; i++) { + let match = matches[i]; + if (host === match || (isRegExp(match) && match.test(host))) { + return true; + } + } + return false; +} + +export default function isURL(url, options) { + assertString(url); + if (!url || /[\s<>]/.test(url)) { + return false; + } + if (url.indexOf('mailto:') === 0) { + return false; + } + options = merge(options, default_url_options); + + if (options.validate_length && url.length >= 2083) { + return false; + } + + if (!options.allow_fragments && url.includes('#')) { + return false; + } + + if (!options.allow_query_components && (url.includes('?') || url.includes('&'))) { + return false; + } + + let protocol, auth, host, hostname, port, port_str, split, ipv6; + + split = url.split('#'); + url = split.shift(); + + split = url.split('?'); + url = split.shift(); + + split = url.split('://'); + if (split.length > 1) { + protocol = split.shift().toLowerCase(); + if (options.require_valid_protocol && options.protocols.indexOf(protocol) === -1) { + return false; + } + } else if (options.require_protocol) { + return false; + } else if (url.slice(0, 2) === '//') { + if (!options.allow_protocol_relative_urls) { + return false; + } + split[0] = url.slice(2); + } + url = split.join('://'); + + if (url === '') { + return false; + } + + split = url.split('/'); + url = split.shift(); + + if (url === '' && !options.require_host) { + return true; + } + + split = url.split('@'); + if (split.length > 1) { + if (options.disallow_auth) { + return false; + } + if (split[0] === '') { + return false; + } + auth = split.shift(); + if (auth.indexOf(':') >= 0 && auth.split(':').length > 2) { + return false; + } + const [user, password] = auth.split(':'); + if (user === '' && password === '') { + return false; + } + } + hostname = split.join('@'); + + port_str = null; + ipv6 = null; + const ipv6_match = hostname.match(wrapped_ipv6); + if (ipv6_match) { + host = ''; + ipv6 = ipv6_match[1]; + port_str = ipv6_match[2] || null; + } else { + split = hostname.split(':'); + host = split.shift(); + if (split.length) { + port_str = split.join(':'); + } + } + + if (port_str !== null && port_str.length > 0) { + port = parseInt(port_str, 10); + if (!/^[0-9]+$/.test(port_str) || port <= 0 || port > 65535) { + return false; + } + } else if (options.require_port) { + return false; + } + + if (options.host_whitelist) { + return checkHost(host, options.host_whitelist); + } + + if (host === '' && !options.require_host) { + return true; + } + + if (!isIP(host) && !isFQDN(host, options) && (!ipv6 || !isIP(ipv6, 6))) { + return false; + } + + host = host || ipv6; + + if (options.host_blacklist && checkHost(host, options.host_blacklist)) { + return false; + } + + return true; +} \ No newline at end of file From a9b97436c54fd78a3e66228b02726cd9b53be642 Mon Sep 17 00:00:00 2001 From: jshemas Date: Mon, 22 Jul 2024 17:37:51 -0400 Subject: [PATCH 2/7] fixing basic linting errors in isUrl --- lib/isURL.ts | 82 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/lib/isURL.ts b/lib/isURL.ts index cba32fe..ba2ccfb 100644 --- a/lib/isURL.ts +++ b/lib/isURL.ts @@ -1,3 +1,9 @@ +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-nocheck +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ // This is from https://github.com/validatorjs/validator.js version: 13.12.0 // https://github.com/validatorjs/validator.js/blob/master/src/lib/util/assertString.js @@ -13,8 +19,20 @@ function assertString(input) { } } +// https://github.com/validatorjs/validator.js/blob/master/src/lib/util/merge.js +// eslint-disable-next-line @typescript-eslint/default-param-last +function merge(obj = { }, defaults) { + // eslint-disable-next-line no-restricted-syntax + for (const key in defaults) { + if (typeof obj[key] === 'undefined') { + obj[key] = defaults[key]; + } + } + return obj; +} + // https://github.com/validatorjs/validator.js/blob/master/src/lib/isFQDN.js -const default_fqdn_options = { +const defaultFqdnOptions = { require_tld: true, allow_underscores: false, allow_trailing_dot: false, @@ -25,7 +43,7 @@ const default_fqdn_options = { function isFQDN(str, options) { assertString(str); - options = merge(options, default_fqdn_options); + options = merge(options, defaultFqdnOptions); /* Remove the optional trailing dot before checking validity */ if (options.allow_trailing_dot && str[str.length - 1] === '.') { @@ -46,7 +64,11 @@ function isFQDN(str, options) { return false; } - if (!options.allow_numeric_tld && !/^([a-z\u00A1-\u00A8\u00AA-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]{2,}|xn[a-z0-9-]{2,})$/i.test(tld)) { + if ( + !options.allow_numeric_tld + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + && !/^([a-z\u00A1-\u00A8\u00AA-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]{2,}|xn[a-z0-9-]{2,})$/i.test(tld) + ) { return false; } @@ -61,6 +83,7 @@ function isFQDN(str, options) { return false; } + // eslint-disable-next-line @typescript-eslint/no-unsafe-return return parts.every((part) => { if (part.length > 63 && !options.ignore_max_length) { return false; @@ -94,16 +117,16 @@ const IPv4AddressFormat = `(${IPv4SegmentFormat}[.]){3}${IPv4SegmentFormat}`; const IPv4AddressRegExp = new RegExp(`^${IPv4AddressFormat}$`); const IPv6SegmentFormat = '(?:[0-9a-fA-F]{1,4})'; -const IPv6AddressRegExp = new RegExp('^(' + - `(?:${IPv6SegmentFormat}:){7}(?:${IPv6SegmentFormat}|:)|` + - `(?:${IPv6SegmentFormat}:){6}(?:${IPv4AddressFormat}|:${IPv6SegmentFormat}|:)|` + - `(?:${IPv6SegmentFormat}:){5}(?::${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,2}|:)|` + - `(?:${IPv6SegmentFormat}:){4}(?:(:${IPv6SegmentFormat}){0,1}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,3}|:)|` + - `(?:${IPv6SegmentFormat}:){3}(?:(:${IPv6SegmentFormat}){0,2}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,4}|:)|` + - `(?:${IPv6SegmentFormat}:){2}(?:(:${IPv6SegmentFormat}){0,3}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,5}|:)|` + - `(?:${IPv6SegmentFormat}:){1}(?:(:${IPv6SegmentFormat}){0,4}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,6}|:)|` + - `(?::((?::${IPv6SegmentFormat}){0,5}:${IPv4AddressFormat}|(?::${IPv6SegmentFormat}){1,7}|:))` + - ')(%[0-9a-zA-Z-.:]{1,})?$'); +const IPv6AddressRegExp = new RegExp('^(' + + `(?:${IPv6SegmentFormat}:){7}(?:${IPv6SegmentFormat}|:)|` + + `(?:${IPv6SegmentFormat}:){6}(?:${IPv4AddressFormat}|:${IPv6SegmentFormat}|:)|` + + `(?:${IPv6SegmentFormat}:){5}(?::${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,2}|:)|` + + `(?:${IPv6SegmentFormat}:){4}(?:(:${IPv6SegmentFormat}){0,1}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,3}|:)|` + + `(?:${IPv6SegmentFormat}:){3}(?:(:${IPv6SegmentFormat}){0,2}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,4}|:)|` + + `(?:${IPv6SegmentFormat}:){2}(?:(:${IPv6SegmentFormat}){0,3}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,5}|:)|` + + `(?:${IPv6SegmentFormat}:){1}(?:(:${IPv6SegmentFormat}){0,4}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,6}|:)|` + + `(?::((?::${IPv6SegmentFormat}){0,5}:${IPv4AddressFormat}|(?::${IPv6SegmentFormat}){1,7}|:))` + + ')(%[0-9a-zA-Z-.:]{1,})?$'); function isIP(str, version = '') { assertString(str); @@ -120,16 +143,6 @@ function isIP(str, version = '') { return false; } -// https://github.com/validatorjs/validator.js/blob/master/src/lib/util/merge.js -function merge(obj = { }, defaults) { - for (const key in defaults) { - if (typeof obj[key] === 'undefined') { - obj[key] = defaults[key]; - } - } - return obj; -} - // https://github.com/validatorjs/validator.js/blob/master/src/lib/isURL.js /* options for isURL method @@ -143,7 +156,7 @@ allow_protocol_relative_urls - if set as true protocol relative URLs will be all validate_length - if set as false isURL will skip string length validation (IE maximum is 2083) */ -const default_url_options = { +const defaultUrlOptions = { protocols: ['http', 'https', 'ftp'], require_tld: true, require_protocol: false, @@ -158,15 +171,16 @@ const default_url_options = { validate_length: true, }; -const wrapped_ipv6 = /^\[([^\]]+)\](?::([0-9]+))?$/; +const wrappedIpv6 = /^\[([^\]]+)\](?::([0-9]+))?$/; function isRegExp(obj) { return Object.prototype.toString.call(obj) === '[object RegExp]'; } function checkHost(host, matches) { - for (let i = 0; i < matches.length; i++) { - let match = matches[i]; + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < matches.length; i += 1) { + const match = matches[i]; if (host === match || (isRegExp(match) && match.test(host))) { return true; } @@ -182,7 +196,7 @@ export default function isURL(url, options) { if (url.indexOf('mailto:') === 0) { return false; } - options = merge(options, default_url_options); + options = merge(options, defaultUrlOptions); if (options.validate_length && url.length >= 2083) { return false; @@ -196,7 +210,9 @@ export default function isURL(url, options) { return false; } - let protocol, auth, host, hostname, port, port_str, split, ipv6; + // eslint-disable-next-line @typescript-eslint/naming-convention + let protocol; let auth; let host; let port; let port_str; let split; let + ipv6; split = url.split('#'); url = split.shift(); @@ -248,13 +264,15 @@ export default function isURL(url, options) { return false; } } - hostname = split.join('@'); + const hostname = split.join('@'); port_str = null; ipv6 = null; - const ipv6_match = hostname.match(wrapped_ipv6); + // eslint-disable-next-line @typescript-eslint/naming-convention + const ipv6_match = hostname.match(wrappedIpv6); if (ipv6_match) { host = ''; + // eslint-disable-next-line prefer-destructuring ipv6 = ipv6_match[1]; port_str = ipv6_match[2] || null; } else { @@ -293,4 +311,4 @@ export default function isURL(url, options) { } return true; -} \ No newline at end of file +} From 8dca3c557198016e8fafe36c96821ca95e94aa7e Mon Sep 17 00:00:00 2001 From: jshemas Date: Mon, 22 Jul 2024 17:41:42 -0400 Subject: [PATCH 3/7] rename file --- lib/{isURL.ts => isUrl.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/{isURL.ts => isUrl.ts} (100%) diff --git a/lib/isURL.ts b/lib/isUrl.ts similarity index 100% rename from lib/isURL.ts rename to lib/isUrl.ts From bf9d4bba5cefeba400c415be08680e8c9079c8c3 Mon Sep 17 00:00:00 2001 From: jshemas Date: Mon, 22 Jul 2024 17:42:49 -0400 Subject: [PATCH 4/7] missing eslint disable is isUrl --- lib/isUrl.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/isUrl.ts b/lib/isUrl.ts index ba2ccfb..509eb5e 100644 --- a/lib/isUrl.ts +++ b/lib/isUrl.ts @@ -132,6 +132,7 @@ function isIP(str, version = '') { assertString(str); version = String(version); if (!version) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return return isIP(str, 4) || isIP(str, 6); } if (version === '4') { From f44a238b409f42eb467095d18548bbf4b97de588 Mon Sep 17 00:00:00 2001 From: jshemas Date: Mon, 22 Jul 2024 17:45:59 -0400 Subject: [PATCH 5/7] replace validator with internal version of isUrl --- lib/utils.ts | 4 ++-- package-lock.json | 18 +----------------- package.json | 4 +--- types/lib/isUrl.d.ts | 1 + types/lib/utils.d.ts | 26 ++++++++++++++++++++++++++ 5 files changed, 31 insertions(+), 22 deletions(-) create mode 100644 types/lib/isUrl.d.ts diff --git a/lib/utils.ts b/lib/utils.ts index 074a2c1..9ecbb04 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -1,4 +1,4 @@ -import validator from 'validator'; +import isUrl from './isUrl'; import type { CustomMetaTags, OgObjectInteral, @@ -30,7 +30,7 @@ export const defaultUrlValidatorSettings = { * */ export function isUrlValid(url: string, urlValidatorSettings: ValidatorSettings): boolean { - return typeof url === 'string' && url.length > 0 && validator.isURL(url, urlValidatorSettings); + return typeof url === 'string' && url.length > 0 && isUrl(url, urlValidatorSettings); } /** diff --git a/package-lock.json b/package-lock.json index 89f42dc..b709dbf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,14 +12,12 @@ "chardet": "^2.0.0", "cheerio": "^1.0.0-rc.12", "iconv-lite": "^0.6.3", - "undici": "^6.19.2", - "validator": "^13.12.0" + "undici": "^6.19.2" }, "devDependencies": { "@snyk/protect": "^1.1292.1", "@types/mocha": "^10.0.7", "@types/node": "^18.19.40", - "@types/validator": "^13.12.0", "@typescript-eslint/eslint-plugin": "^7.16.1", "@typescript-eslint/parser": "^7.16.1", "chai": "^4.4.1", @@ -852,12 +850,6 @@ "undici-types": "~5.26.4" } }, - "node_modules/@types/validator": { - "version": "13.12.0", - "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.12.0.tgz", - "integrity": "sha512-nH45Lk7oPIJ1RVOF6JgFI6Dy0QpHEzq4QecZhvguxYPDwT8c93prCMqAtiIttm39voZ+DDR+qkNnMpJmMBRqag==", - "dev": true - }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "7.16.1", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.16.1.tgz", @@ -5569,14 +5561,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/validator": { - "version": "13.12.0", - "resolved": "https://registry.npmjs.org/validator/-/validator-13.12.0.tgz", - "integrity": "sha512-c1Q0mCiPlgdTVVVIJIrBuxNicYE+t/7oKeI9MWLj3fh/uq2Pxh/3eeWbVZ4OcGW1TUf53At0njHw5SMdA3tmMg==", - "engines": { - "node": ">= 0.10" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index ebb715b..47e1396 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,7 @@ "chardet": "^2.0.0", "cheerio": "^1.0.0-rc.12", "iconv-lite": "^0.6.3", - "undici": "^6.19.2", - "validator": "^13.12.0" + "undici": "^6.19.2" }, "files": [ "/dist", @@ -51,7 +50,6 @@ "@snyk/protect": "^1.1292.1", "@types/mocha": "^10.0.7", "@types/node": "^18.19.40", - "@types/validator": "^13.12.0", "@typescript-eslint/eslint-plugin": "^7.16.1", "@typescript-eslint/parser": "^7.16.1", "chai": "^4.4.1", diff --git a/types/lib/isUrl.d.ts b/types/lib/isUrl.d.ts new file mode 100644 index 0000000..46ec2f3 --- /dev/null +++ b/types/lib/isUrl.d.ts @@ -0,0 +1 @@ +export default function isURL(url: any, options: any): boolean; diff --git a/types/lib/utils.d.ts b/types/lib/utils.d.ts index d2a6bc6..26a185c 100644 --- a/types/lib/utils.d.ts +++ b/types/lib/utils.d.ts @@ -83,3 +83,29 @@ export declare function optionSetup(ogsOptions: OpenGraphScraperOptions): { * */ export declare function isCustomMetaTagsValid(customMetaTags: CustomMetaTags[]): boolean; +/** + * Unescape script text. + * + * Certain websites escape script text within script tags, which can + * interfere with `JSON.parse()`. Therefore, we need to unescape it. + * + * Known good escape sequences: + * + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions/Character_escape#uhhhh + * + * ```js + * JSON.parse('"\\u2611"'); // '☑' + * ``` + * + * Known bad escape sequences: + * + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions/Character_escape#xhh + * + * ```js + * JSON.parse('"\\x26"'); // '&' + * ``` + * + * @param {string} scriptText - the text of the script tag + * @returns {string} unescaped script text + */ +export declare function unescapeScriptText(scriptText: string): string; From 6999e58bfc4a635adbf719a53c36b76cb147541a Mon Sep 17 00:00:00 2001 From: jshemas Date: Mon, 22 Jul 2024 17:48:56 -0400 Subject: [PATCH 6/7] updating dependencies --- package-lock.json | 142 +++++++++++++++++++++++----------------------- package.json | 10 ++-- 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/package-lock.json b/package-lock.json index b709dbf..cd27b5b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,21 +12,21 @@ "chardet": "^2.0.0", "cheerio": "^1.0.0-rc.12", "iconv-lite": "^0.6.3", - "undici": "^6.19.2" + "undici": "^6.19.4" }, "devDependencies": { "@snyk/protect": "^1.1292.1", "@types/mocha": "^10.0.7", - "@types/node": "^18.19.40", - "@typescript-eslint/eslint-plugin": "^7.16.1", - "@typescript-eslint/parser": "^7.16.1", + "@types/node": "^18.19.41", + "@typescript-eslint/eslint-plugin": "^7.17.0", + "@typescript-eslint/parser": "^7.17.0", "chai": "^4.4.1", "eslint": "^8.57.0", "eslint-config-airbnb-base": "^15.0.0", "eslint-config-airbnb-typescript": "^18.0.0", "eslint-plugin-import": "^2.29.1", "eslint-plugin-mocha": "^10.4.3", - "eslint-plugin-promise": "^6.4.0", + "eslint-plugin-promise": "^6.6.0", "mocha": "^10.6.0", "nyc": "^17.0.0", "sinon": "^18.0.0", @@ -842,25 +842,25 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.19.40", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.40.tgz", - "integrity": "sha512-MIxieZHrm4Ee8XArBIc+Or9HINt2StOmCbgRcXGSJl8q14svRvkZPe7LJq9HKtTI1SK3wU8b91TjntUm7T69Pg==", + "version": "18.19.41", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.41.tgz", + "integrity": "sha512-LX84pRJ+evD2e2nrgYCHObGWkiQJ1mL+meAgbvnwk/US6vmMY7S2ygBTGV2Jw91s9vUsLSXeDEkUHZIJGLrhsg==", "dev": true, "dependencies": { "undici-types": "~5.26.4" } }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.16.1.tgz", - "integrity": "sha512-SxdPak/5bO0EnGktV05+Hq8oatjAYVY3Zh2bye9pGZy6+jwyR3LG3YKkV4YatlsgqXP28BTeVm9pqwJM96vf2A==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.17.0.tgz", + "integrity": "sha512-pyiDhEuLM3PuANxH7uNYan1AaFs5XE0zw1hq69JBvGvE7gSuEoQl1ydtEe/XQeoC3GQxLXyOVa5kNOATgM638A==", "dev": true, "dependencies": { "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "7.16.1", - "@typescript-eslint/type-utils": "7.16.1", - "@typescript-eslint/utils": "7.16.1", - "@typescript-eslint/visitor-keys": "7.16.1", + "@typescript-eslint/scope-manager": "7.17.0", + "@typescript-eslint/type-utils": "7.17.0", + "@typescript-eslint/utils": "7.17.0", + "@typescript-eslint/visitor-keys": "7.17.0", "graphemer": "^1.4.0", "ignore": "^5.3.1", "natural-compare": "^1.4.0", @@ -884,15 +884,15 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.16.1.tgz", - "integrity": "sha512-u+1Qx86jfGQ5i4JjK33/FnawZRpsLxRnKzGE6EABZ40KxVT/vWsiZFEBBHjFOljmmV3MBYOHEKi0Jm9hbAOClA==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.17.0.tgz", + "integrity": "sha512-puiYfGeg5Ydop8eusb/Hy1k7QmOU6X3nvsqCgzrB2K4qMavK//21+PzNE8qeECgNOIoertJPUC1SpegHDI515A==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "7.16.1", - "@typescript-eslint/types": "7.16.1", - "@typescript-eslint/typescript-estree": "7.16.1", - "@typescript-eslint/visitor-keys": "7.16.1", + "@typescript-eslint/scope-manager": "7.17.0", + "@typescript-eslint/types": "7.17.0", + "@typescript-eslint/typescript-estree": "7.17.0", + "@typescript-eslint/visitor-keys": "7.17.0", "debug": "^4.3.4" }, "engines": { @@ -912,13 +912,13 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.16.1.tgz", - "integrity": "sha512-nYpyv6ALte18gbMz323RM+vpFpTjfNdyakbf3nsLvF43uF9KeNC289SUEW3QLZ1xPtyINJ1dIsZOuWuSRIWygw==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.17.0.tgz", + "integrity": "sha512-0P2jTTqyxWp9HiKLu/Vemr2Rg1Xb5B7uHItdVZ6iAenXmPo4SZ86yOPCJwMqpCyaMiEHTNqizHfsbmCFT1x9SA==", "dev": true, "dependencies": { - "@typescript-eslint/types": "7.16.1", - "@typescript-eslint/visitor-keys": "7.16.1" + "@typescript-eslint/types": "7.17.0", + "@typescript-eslint/visitor-keys": "7.17.0" }, "engines": { "node": "^18.18.0 || >=20.0.0" @@ -929,13 +929,13 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.16.1.tgz", - "integrity": "sha512-rbu/H2MWXN4SkjIIyWcmYBjlp55VT+1G3duFOIukTNFxr9PI35pLc2ydwAfejCEitCv4uztA07q0QWanOHC7dA==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.17.0.tgz", + "integrity": "sha512-XD3aaBt+orgkM/7Cei0XNEm1vwUxQ958AOLALzPlbPqb8C1G8PZK85tND7Jpe69Wualri81PLU+Zc48GVKIMMA==", "dev": true, "dependencies": { - "@typescript-eslint/typescript-estree": "7.16.1", - "@typescript-eslint/utils": "7.16.1", + "@typescript-eslint/typescript-estree": "7.17.0", + "@typescript-eslint/utils": "7.17.0", "debug": "^4.3.4", "ts-api-utils": "^1.3.0" }, @@ -956,9 +956,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.16.1.tgz", - "integrity": "sha512-AQn9XqCzUXd4bAVEsAXM/Izk11Wx2u4H3BAfQVhSfzfDOm/wAON9nP7J5rpkCxts7E5TELmN845xTUCQrD1xIQ==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.17.0.tgz", + "integrity": "sha512-a29Ir0EbyKTKHnZWbNsrc/gqfIBqYPwj3F2M+jWE/9bqfEHg0AMtXzkbUkOG6QgEScxh2+Pz9OXe11jHDnHR7A==", "dev": true, "engines": { "node": "^18.18.0 || >=20.0.0" @@ -969,13 +969,13 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.16.1.tgz", - "integrity": "sha512-0vFPk8tMjj6apaAZ1HlwM8w7jbghC8jc1aRNJG5vN8Ym5miyhTQGMqU++kuBFDNKe9NcPeZ6x0zfSzV8xC1UlQ==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.17.0.tgz", + "integrity": "sha512-72I3TGq93t2GoSBWI093wmKo0n6/b7O4j9o8U+f65TVD0FS6bI2180X5eGEr8MA8PhKMvYe9myZJquUT2JkCZw==", "dev": true, "dependencies": { - "@typescript-eslint/types": "7.16.1", - "@typescript-eslint/visitor-keys": "7.16.1", + "@typescript-eslint/types": "7.17.0", + "@typescript-eslint/visitor-keys": "7.17.0", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -997,15 +997,15 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.16.1.tgz", - "integrity": "sha512-WrFM8nzCowV0he0RlkotGDujx78xudsxnGMBHI88l5J8wEhED6yBwaSLP99ygfrzAjsQvcYQ94quDwI0d7E1fA==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.17.0.tgz", + "integrity": "sha512-r+JFlm5NdB+JXc7aWWZ3fKSm1gn0pkswEwIYsrGPdsT2GjsRATAKXiNtp3vgAAO1xZhX8alIOEQnNMl3kbTgJw==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "7.16.1", - "@typescript-eslint/types": "7.16.1", - "@typescript-eslint/typescript-estree": "7.16.1" + "@typescript-eslint/scope-manager": "7.17.0", + "@typescript-eslint/types": "7.17.0", + "@typescript-eslint/typescript-estree": "7.17.0" }, "engines": { "node": "^18.18.0 || >=20.0.0" @@ -1019,12 +1019,12 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "7.16.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.16.1.tgz", - "integrity": "sha512-Qlzzx4sE4u3FsHTPQAAQFJFNOuqtuY0LFrZHwQ8IHK705XxBiWOFkfKRWu6niB7hwfgnwIpO4jTC75ozW1PHWg==", + "version": "7.17.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.17.0.tgz", + "integrity": "sha512-RVGC9UhPOCsfCdI9pU++K4nD7to+jTcMIbXTSOcrLqUEW6gF2pU1UUbYJKc9cvcRSK1UDeMJ7pdMxf4bhMpV/A==", "dev": true, "dependencies": { - "@typescript-eslint/types": "7.16.1", + "@typescript-eslint/types": "7.17.0", "eslint-visitor-keys": "^3.4.3" }, "engines": { @@ -1458,9 +1458,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001642", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001642.tgz", - "integrity": "sha512-3XQ0DoRgLijXJErLSl+bLnJ+Et4KqV1PY6JJBGAFlsNsz31zeAIncyeZfLCabHK/jtSh+671RM9YMldxjUPZtA==", + "version": "1.0.30001643", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001643.tgz", + "integrity": "sha512-ERgWGNleEilSrHM6iUz/zJNSQTP8Mr21wDWpdgvRwcTXGAq6jMtOUPP4dqFPTdKqZ2wKTdtB+uucZ3MRpAUSmg==", "dev": true, "funding": [ { @@ -1931,9 +1931,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.828", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.828.tgz", - "integrity": "sha512-QOIJiWpQJDHAVO4P58pwb133Cwee0nbvy/MV1CwzZVGpkH1RX33N3vsaWRCpR6bF63AAq366neZrRTu7Qlsbbw==", + "version": "1.4.832", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.832.tgz", + "integrity": "sha512-cTen3SB0H2SGU7x467NRe1eVcQgcuS6jckKfWJHia2eo0cHIGOqHoAxevIYZD4eRHcWjkvFzo93bi3vJ9W+1lA==", "dev": true }, "node_modules/emoji-regex": { @@ -2357,9 +2357,9 @@ } }, "node_modules/eslint-plugin-promise": { - "version": "6.4.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-6.4.0.tgz", - "integrity": "sha512-/KWWRaD3fGkVCZsdR0RU53PSthFmoHVhZl+y9+6DqeDLSikLdlUVpVEAmI6iCRR5QyOjBYBqHZV/bdv4DJ4Gtw==", + "version": "6.6.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-6.6.0.tgz", + "integrity": "sha512-57Zzfw8G6+Gq7axm2Pdo3gW/Rx3h9Yywgn61uE/3elTCOePEHVrn2i5CdfBwA1BLK0Q0WqctICIUSqXZW/VprQ==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -3243,9 +3243,9 @@ } }, "node_modules/is-core-module": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.14.0.tgz", - "integrity": "sha512-a5dFJih5ZLYlRtDc0dZWP7RiKr6xIKzmn/oAYCDvdLThadVgyJwlaoQPmRtMSpz+rk0OGAgIu+TcM9HUF0fk1A==", + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.0.tgz", + "integrity": "sha512-Dd+Lb2/zvk9SKy1TGCt1wFJFo/MWBPMX5x7KcvLajWTGuomczdQX61PvY5yK6SVACwpoexWo81IfFyoKY2QnTA==", "dev": true, "dependencies": { "hasown": "^2.0.2" @@ -3864,9 +3864,9 @@ } }, "node_modules/mocha": { - "version": "10.6.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.6.0.tgz", - "integrity": "sha512-hxjt4+EEB0SA0ZDygSS015t65lJw/I2yRCS3Ae+SJ5FrbzrXgfYwJr96f0OvIXdj7h4lv/vLCrH3rkiuizFSvw==", + "version": "10.7.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.7.0.tgz", + "integrity": "sha512-v8/rBWr2VO5YkspYINnvu81inSz2y3ODJrhO175/Exzor1RcEZZkizgE2A+w/CAXXoESS8Kys5E62dOHGHzULA==", "dev": true, "dependencies": { "ansi-colors": "^4.1.3", @@ -3969,9 +3969,9 @@ } }, "node_modules/node-releases": { - "version": "2.0.17", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.17.tgz", - "integrity": "sha512-Ww6ZlOiEQfPfXM45v17oabk77Z7mg5bOt7AjDyzy7RjK9OrLrLC8dyZQoAPEOtFX9SaNf1Tdvr5gRJWdTJj7GA==", + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz", + "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==", "dev": true }, "node_modules/normalize-path": { @@ -5500,9 +5500,9 @@ } }, "node_modules/undici": { - "version": "6.19.2", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.19.2.tgz", - "integrity": "sha512-JfjKqIauur3Q6biAtHJ564e3bWa8VvT+7cSiOJHFbX4Erv6CLGDpg8z+Fmg/1OI/47RA+GI2QZaF48SSaLvyBA==", + "version": "6.19.4", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.19.4.tgz", + "integrity": "sha512-i3uaEUwNdkRq2qtTRRJb13moW5HWqviu7Vl7oYRYz++uPtGHJj+x7TGjcEuwS5Mt2P4nA0U9dhIX3DdB6JGY0g==", "engines": { "node": ">=18.17" } diff --git a/package.json b/package.json index 47e1396..beb0296 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "chardet": "^2.0.0", "cheerio": "^1.0.0-rc.12", "iconv-lite": "^0.6.3", - "undici": "^6.19.2" + "undici": "^6.19.4" }, "files": [ "/dist", @@ -49,16 +49,16 @@ "devDependencies": { "@snyk/protect": "^1.1292.1", "@types/mocha": "^10.0.7", - "@types/node": "^18.19.40", - "@typescript-eslint/eslint-plugin": "^7.16.1", - "@typescript-eslint/parser": "^7.16.1", + "@types/node": "^18.19.41", + "@typescript-eslint/eslint-plugin": "^7.17.0", + "@typescript-eslint/parser": "^7.17.0", "chai": "^4.4.1", "eslint": "^8.57.0", "eslint-config-airbnb-base": "^15.0.0", "eslint-config-airbnb-typescript": "^18.0.0", "eslint-plugin-import": "^2.29.1", "eslint-plugin-mocha": "^10.4.3", - "eslint-plugin-promise": "^6.4.0", + "eslint-plugin-promise": "^6.6.0", "mocha": "^10.6.0", "nyc": "^17.0.0", "sinon": "^18.0.0", From edef1f310bf96e6530b282ecc0cd2e9742e32d50 Mon Sep 17 00:00:00 2001 From: jshemas Date: Mon, 22 Jul 2024 17:59:22 -0400 Subject: [PATCH 7/7] updating changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54b80fd..bf91c88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Change Log +## 6.7.0 + +- Replace `validator` with internal version of `isUrl` so we have better control on how that works. +- Fix issue where `JSON` parsing fails when Youtube escape '&' to '\x26'. +- Updating dependencies + ## 6.6.3 - Fix issue with the `charset` fallback. Replace Buffer.from with Uint8Array since body is always html