diff --git a/campaigns-schema.yaml b/campaigns-schema.yaml index f9f3079..5d500b8 100644 --- a/campaigns-schema.yaml +++ b/campaigns-schema.yaml @@ -30,6 +30,10 @@ deployments: required: false type: boolean description: Whether a .txt file should be processed as a .ict file. + filter_kmz: + required: false + type: boolean + description: Some zip files contain a kmz file that is the only file needed. In that case, it will only extract the kmz and uncompress it, resulting in a kml file. coords_divisor: required: false type: integer diff --git a/campaigns/PODEX/deployments.yaml b/campaigns/PODEX/deployments.yaml new file mode 100644 index 0000000..e28c77e --- /dev/null +++ b/campaigns/PODEX/deployments.yaml @@ -0,0 +1,18 @@ +--- +deployments: +- name: PODEX-D1_2013 + platforms: + - name: ER-2 + files: + - https://asp-archive.arc.nasa.gov/PODEX/N809NA/2013-01-14/IWG1.xml + - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-14/IWG1.15Jan2013-0002 + - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-16/IWG1.16Jan2013-2108 + - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-18/IWG1.18Jan2013-2203 + - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-20/IWG1.20Jan2013-2149 + - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-22/IWG1.23Jan2013-0030 + - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-28/IWG1.28Jan2013-2322 + - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-31/IWG1.31Jan2013-2257 + - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-02-01/IWG1.01Feb2013-2317 + - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-02-03/IWG1.03Feb2013-1452 + - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-02-03/IWG1.03Feb2013-2312 + - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-02-06/IWG1.07Feb2013-0001 \ No newline at end of file diff --git a/campaigns/Polar Winds/deployments.yaml b/campaigns/Polar Winds/deployments.yaml new file mode 100644 index 0000000..27389f2 --- /dev/null +++ b/campaigns/Polar Winds/deployments.yaml @@ -0,0 +1,37 @@ +--- +deployments: +- name: Polar Winds-D1_2014 + platforms: + - name: B-200 + filter_kmz: true + files: + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141029-115154_txt.zip + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141030-132302_txt.zip + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141031-110108_txt.zip + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141103-135618_txt.zip + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141104-135816_txt.zip + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141105-141140_txt.zip + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141106-140534_txt.zip + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141107-141738_txt.zip + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141108-155441_txt.zip + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141110-142837_txt.zip + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141111-140700_txt.zip + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141112-110621_txt.zip + - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141113-140215_txt.zip +- name: Polar Winds-D2_2015 + platforms: + - name: DC-8 + files: + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2014-10-25/IWG1.xml + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-07/IWG1.07May2015-2226 + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-09/IWG1.09May2015-1401 + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-11/IWG1.11May2015-1728 + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-13/IWG1.13May2015-1509 + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-15/IWG1.15May2015-2032 + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-16/IWG1.16May2015-2115 + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-17/IWG1.17May2015-2117 + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-19/IWG1.19May2015-1708 + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-21/IWG1.21May2015-2202 + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-23/IWG1.23May2015-2001 + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-24/IWG1.24May2015-1835 + - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-25/IWG1.25May2015-1729 \ No newline at end of file diff --git a/task/get-files-list.js b/task/get-files-list.js new file mode 100644 index 0000000..bc221ea --- /dev/null +++ b/task/get-files-list.js @@ -0,0 +1,16 @@ +const { listLinks } = require('./src/list-links'); + +const url = process.argv[2]; +const outputFile = process.argv[3]; + +// this is a utility command to list the files URLs in some NASA websites, +// like https://asp-archive.arc.nasa.gov/PODEX/N809NA/ +// It will create a txt with the URLs of the files listed in the +// subpages of that webpage. +// It also filters out all the files that don't contain the string "IWG1." +// and the .xml files + +// Usage: +// yarn list-files https://asp-archive.arc.nasa.gov/PODEX/N809NA/ podex.txt + +listLinks(url, outputFile); diff --git a/task/package.json b/task/package.json index 8a295be..3b3eaec 100644 --- a/task/package.json +++ b/task/package.json @@ -6,6 +6,7 @@ "scripts": { "download": "node downloadCampaign.js", "lint": "eslint --ext .js", + "list-files": "node get-files-list.js", "process": "node processAll.js", "test": "jest", "xls2csv": "node xls2csv.js" @@ -13,7 +14,10 @@ "dependencies": { "@mapbox/geojson-merge": "^1.1.1", "@tmcw/togeojson": "^5.8.1", - "@turf/distance": "^6.5.0", + "@turf/distance": "^7.1.0", + "@turf/helpers": "^7.1.0", + "axios": "^1.7.7", + "cheerio": "^1.0.0", "csv2geojson": "^5.1.2", "d3-dsv": "1.0.1", "download": "8.0.0", diff --git a/task/src/convert-kml.js b/task/src/convert-kml.js index e89c957..ed10d91 100644 --- a/task/src/convert-kml.js +++ b/task/src/convert-kml.js @@ -2,15 +2,23 @@ const toGeojson = require('@tmcw/togeojson'); const fs = require('fs'); const { DOMParser } = require('xmldom'); const unzipper = require('unzipper'); +const { lineString } = require('@turf/helpers'); const simplify = require('simplify-geojson'); const kml2geojson = (filePath, properties) => { const file = fs.readFileSync(filePath); const kml = new DOMParser().parseFromString(file.toString()); const geojson = toGeojson.kml(kml); - const features = geojson.features - .filter((f) => f.geometry.type === 'LineString') - .map((f) => ({ ...f, properties })); + let features; + // some kml files have all features as points, in that case, + // we need to convert it to a single LineString + if (geojson.features.every((f) => f.geometry.type === 'Point')) { + features = [lineString(geojson.features.map((f) => f.geometry.coordinates), properties)]; + } else { + features = geojson.features + .filter((f) => f.geometry.type === 'LineString') + .map((f) => ({ ...f, properties })); + } return simplify({ ...geojson, features }, 0.001); }; @@ -37,7 +45,32 @@ const kmz2kml = (filePath) => new Promise((resolve, reject) => { .on('error', reject); }); +const extractKmlContent = (filePath) => new Promise((resolve, reject) => { + fs.createReadStream(filePath) + .pipe(unzipper.ParseOne(/\.kmz$/)) + .pipe(unzipper.Parse()) + .on('entry', (entry) => { + if (entry.path.indexOf('.kml') === -1) { + entry.autodrain(); + return; + } + let data = ''; + + entry.on('error', reject); + + entry.on('data', (chunk) => { + data += chunk; + }); + + entry.on('end', () => { + resolve(data); + }); + }) + .on('error', reject); +}); + module.exports = { kml2geojson, + extractKmlContent, kmz2kml, }; diff --git a/task/src/download.js b/task/src/download.js index 3f1612b..ad8d284 100644 --- a/task/src/download.js +++ b/task/src/download.js @@ -3,11 +3,11 @@ const path = require('path'); const unzipper = require('unzipper'); const download = require('download'); const { getPlatformConfig, readCampaignYaml, urlHasFileExtension } = require('./utils'); -const { kmz2kml } = require('./convert-kml'); +const { kmz2kml, extractKmlContent } = require('./convert-kml'); const replaceSlash = (str) => str.replaceAll('/', '-'); -const downloadFile = async (url, dir) => { +const downloadFile = async (url, dir, platformConfig) => { await download( url, dir, @@ -16,8 +16,14 @@ const downloadFile = async (url, dir) => { // if the file is a zip, decompress it if (url.endsWith('.zip')) { const filePath = path.join(dir, path.basename(url)); - const zip = await unzipper.Open.file(filePath); - await zip.extract({ path: dir }); + // some zip files, have only a kmz file that needs to be extracted as kml + if (platformConfig.filter_kmz) { + const kml = await extractKmlContent(filePath); + fs.writeFileSync(filePath.replace('.zip', '.kml'), kml); + } else { + const zip = await unzipper.Open.file(filePath); + await zip.extract({ path: dir }); + } fs.unlinkSync(filePath); } // The GRIP campaign has files without an extension and others with .dat that should be txt @@ -62,7 +68,9 @@ const downloadPlatform = async (campaignPath, deployment, platform, files) => { const platformPath = path.join(campaignPath, deployment, platform); const platformConfig = getPlatformConfig(platformPath); await createDir(platformPath); - await Promise.all(files.map((file) => downloadFile(file, platformPath))); + await Promise.all( + files.map((file) => downloadFile(file, platformPath, platformConfig)) + ); // some .ict files have a .ER2 or .WB57 extension, // so we need to rename it after the download if (platformConfig.rename_as_ict) { diff --git a/task/src/list-links.js b/task/src/list-links.js new file mode 100644 index 0000000..21b960f --- /dev/null +++ b/task/src/list-links.js @@ -0,0 +1,38 @@ +const fs = require('fs'); +const axios = require('axios'); +const cheerio = require('cheerio'); + +async function getLinks(url) { + /** + * Retrieves all the links from a given webpage. + * @param {string} url - The URL of the webpage to be accessed. + * @returns {string[]} - A list of all the links found on the webpage. + */ + const response = await axios.get(url); + const $ = cheerio.load(response.data); + const links = $('a').map((_, link) => `${url}/${$(link).attr('href')}`).get(); + return links; +} + +function saveToCSV(links, outputFileName) { + /** + * Saves a list of links to a CSV file. + * @param {string[]} links - A list of links to be saved. + * @param {string} outputFileName - The name of the output CSV file. + */ + fs.writeFileSync(outputFileName, links.join('\n')); +} + +async function listLinks(url, outputFileName) { + const initialLinks = await getLinks(url); + const allLinks = await Promise.all(initialLinks.map((i) => getLinks(i))); + saveToCSV( + allLinks + .flat().filter((i) => i.includes('IWG1.') && !i.includes('.xml')), + outputFileName + ); +} + +module.exports = { + listLinks, +}; diff --git a/task/src/utils.js b/task/src/utils.js index 4f2ece4..cf5cb5f 100644 --- a/task/src/utils.js +++ b/task/src/utils.js @@ -65,8 +65,7 @@ const divideCoordinates = (features, coordsDivisor) => features.map((i) => { }); const urlHasFileExtension = (url) => { - // eslint-disable-next-line no-unused-vars - const [name, ext] = path.basename(url).split('.'); + const ext = path.basename(url).split('.').slice(-1)[0]; return ext && [3, 4].includes(ext.length); }; diff --git a/task/tests/utils.test.js b/task/tests/utils.test.js index a2bc5ff..761be69 100644 --- a/task/tests/utils.test.js +++ b/task/tests/utils.test.js @@ -33,6 +33,9 @@ describe('getPlatformConfig', () => { describe('urlHasFileExtension', () => { it('return true if an URL ends with any 3 or 4 letters file extension', () => { + expect(urlHasFileExtension( + 'https://nasa.gov/PolarWindsI_DAWN_KingAirUC-12B_1.20141113-140215_txt.zip' + )).toBeTruthy(); expect(urlHasFileExtension('https://a.com/b/c/d.ext')).toBeTruthy(); expect(urlHasFileExtension('https://a.com/b/c/d.WB57')).toBeTruthy(); expect(urlHasFileExtension('https://a.com/b/c/d.WB57')).toBeTruthy(); diff --git a/task/tests/validate-yaml.test.js b/task/tests/validate-yaml.test.js index bf2107a..1ef6af3 100644 --- a/task/tests/validate-yaml.test.js +++ b/task/tests/validate-yaml.test.js @@ -20,6 +20,7 @@ const structure = array( tsv_format: optional(boolean()), rename_as_ict: optional(boolean()), process_as_ict: optional(boolean()), + filter_kmz: optional(boolean()), coords_divisor: optional(number()), data_start_line_fix: optional(number()), header_content: optional(string()), diff --git a/task/yarn.lock b/task/yarn.lock index 1f372f5..c394822 100644 --- a/task/yarn.lock +++ b/task/yarn.lock @@ -679,25 +679,32 @@ resolved "https://registry.yarnpkg.com/@tmcw/togeojson/-/togeojson-5.8.1.tgz#6cbcc9b1484ed28e71bdd4b5f96ceae540a4533f" integrity sha512-2YNrbis3l5kS0XrYwiHEZcGwiRp0MJ5CvwGwtMWp2z2tsVlskeec2qgvKHnF0RCwI5GnjrrBOoKsWfndEnd3LA== -"@turf/distance@^6.5.0": - version "6.5.0" - resolved "https://registry.yarnpkg.com/@turf/distance/-/distance-6.5.0.tgz#21f04d5f86e864d54e2abde16f35c15b4f36149a" - integrity sha512-xzykSLfoURec5qvQJcfifw/1mJa+5UwByZZ5TZ8iaqjGYN0vomhV9aiSLeYdUGtYRESZ+DYC/OzY+4RclZYgMg== +"@turf/distance@^7.1.0": + version "7.1.0" + resolved "https://registry.yarnpkg.com/@turf/distance/-/distance-7.1.0.tgz#12317573f9774e1c9d6fb08bf90338d9904d2f1d" + integrity sha512-hhNHhxCHB3ddzAGCNY4BtE29OZh+DAJPvUapQz+wOjISnlwvMcwLKvslgHWSYF536QDVe/93FEU2q67+CsZTPA== dependencies: - "@turf/helpers" "^6.5.0" - "@turf/invariant" "^6.5.0" + "@turf/helpers" "^7.1.0" + "@turf/invariant" "^7.1.0" + "@types/geojson" "^7946.0.10" + tslib "^2.6.2" -"@turf/helpers@^6.5.0": - version "6.5.0" - resolved "https://registry.yarnpkg.com/@turf/helpers/-/helpers-6.5.0.tgz#f79af094bd6b8ce7ed2bd3e089a8493ee6cae82e" - integrity sha512-VbI1dV5bLFzohYYdgqwikdMVpe7pJ9X3E+dlr425wa2/sMJqYDhTO++ec38/pcPvPE6oD9WEEeU3Xu3gza+VPw== +"@turf/helpers@^7.1.0": + version "7.1.0" + resolved "https://registry.yarnpkg.com/@turf/helpers/-/helpers-7.1.0.tgz#eb734e291c9c205822acdd289fe20e91c3cb1641" + integrity sha512-dTeILEUVeNbaEeoZUOhxH5auv7WWlOShbx7QSd4s0T4Z0/iz90z9yaVCtZOLbU89umKotwKaJQltBNO9CzVgaQ== + dependencies: + "@types/geojson" "^7946.0.10" + tslib "^2.6.2" -"@turf/invariant@^6.5.0": - version "6.5.0" - resolved "https://registry.yarnpkg.com/@turf/invariant/-/invariant-6.5.0.tgz#970afc988023e39c7ccab2341bd06979ddc7463f" - integrity sha512-Wv8PRNCtPD31UVbdJE/KVAWKe7l6US+lJItRR/HOEW3eh+U/JwRCSUl/KZ7bmjM/C+zLNoreM2TU6OoLACs4eg== +"@turf/invariant@^7.1.0": + version "7.1.0" + resolved "https://registry.yarnpkg.com/@turf/invariant/-/invariant-7.1.0.tgz#c90cffa093291316b597212396d68bf9e465cf2e" + integrity sha512-OCLNqkItBYIP1nE9lJGuIUatWGtQ4rhBKAyTfFu0z8npVzGEYzvguEeof8/6LkKmTTEHW53tCjoEhSSzdRh08Q== dependencies: - "@turf/helpers" "^6.5.0" + "@turf/helpers" "^7.1.0" + "@types/geojson" "^7946.0.10" + tslib "^2.6.2" "@types/babel__core@^7.1.14": version "7.20.5" @@ -732,6 +739,11 @@ dependencies: "@babel/types" "^7.20.7" +"@types/geojson@^7946.0.10": + version "7946.0.15" + resolved "https://registry.yarnpkg.com/@types/geojson/-/geojson-7946.0.15.tgz#f9d55fd5a0aa2de9dc80b1b04e437538b7298868" + integrity sha512-9oSxFzDCT2Rj6DfcHF8G++jxBKS7mBqXl5xrRW+Kbvjry6Uduya2iiwqHPhVXpasAVMBYKkEPGgKhd3+/HZ6xA== + "@types/graceful-fs@^4.1.3": version "4.1.9" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.9.tgz#2a06bc0f68a20ab37b3e36aa238be6abdf49e8b4" @@ -973,6 +985,11 @@ arrify@^1.0.1: resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d" integrity sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA== +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== + available-typed-arrays@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz#a5cc375d6a03c2efc87a553f3e0b1522def14846" @@ -980,6 +997,15 @@ available-typed-arrays@^1.0.7: dependencies: possible-typed-array-names "^1.0.0" +axios@^1.7.7: + version "1.7.7" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.7.tgz#2f554296f9892a72ac8d8e4c5b79c14a91d0a47f" + integrity sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q== + dependencies: + follow-redirects "^1.15.6" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + babel-jest@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-29.7.0.tgz#f4369919225b684c56085998ac63dbd05be020d5" @@ -1063,6 +1089,11 @@ bluebird@~3.7.2: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== +boolbase@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" + integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww== + brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" @@ -1214,6 +1245,35 @@ char-regex@^1.0.2: resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw== +cheerio-select@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/cheerio-select/-/cheerio-select-2.1.0.tgz#4d8673286b8126ca2a8e42740d5e3c4884ae21b4" + integrity sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g== + dependencies: + boolbase "^1.0.0" + css-select "^5.1.0" + css-what "^6.1.0" + domelementtype "^2.3.0" + domhandler "^5.0.3" + domutils "^3.0.1" + +cheerio@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0.tgz#1ede4895a82f26e8af71009f961a9b8cb60d6a81" + integrity sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww== + dependencies: + cheerio-select "^2.1.0" + dom-serializer "^2.0.0" + domhandler "^5.0.3" + domutils "^3.1.0" + encoding-sniffer "^0.2.0" + htmlparser2 "^9.1.0" + parse5 "^7.1.2" + parse5-htmlparser2-tree-adapter "^7.0.0" + parse5-parser-stream "^7.1.2" + undici "^6.19.5" + whatwg-mimetype "^4.0.0" + ci-info@^3.2.0: version "3.9.0" resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.9.0.tgz#4279a62028a7b1f262f3473fc9605f5e218c59b4" @@ -1279,6 +1339,13 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +combined-stream@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + commander@2.19.0: version "2.19.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a" @@ -1366,6 +1433,22 @@ cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" +css-select@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/css-select/-/css-select-5.1.0.tgz#b8ebd6554c3637ccc76688804ad3f6a6fdaea8a6" + integrity sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg== + dependencies: + boolbase "^1.0.0" + css-what "^6.1.0" + domhandler "^5.0.2" + domutils "^3.0.1" + nth-check "^2.0.1" + +css-what@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4" + integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw== + csv2geojson@^5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/csv2geojson/-/csv2geojson-5.1.2.tgz#4869e48d26f849fbbe93f90b76094fb1861b7c72" @@ -1525,6 +1608,11 @@ defined@~1.0.1: resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.1.tgz#c0b9db27bfaffd95d6f61399419b893df0f91ebf" integrity sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q== +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== + detect-newline@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" @@ -1549,6 +1637,36 @@ doctrine@^3.0.0: dependencies: esutils "^2.0.2" +dom-serializer@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-2.0.0.tgz#e41b802e1eedf9f6cae183ce5e622d789d7d8e53" + integrity sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.2" + entities "^4.2.0" + +domelementtype@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d" + integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== + +domhandler@^5.0.2, domhandler@^5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31" + integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w== + dependencies: + domelementtype "^2.3.0" + +domutils@^3.0.1, domutils@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.1.0.tgz#c47f551278d3dc4b0b1ab8cbb42d751a6f0d824e" + integrity sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA== + dependencies: + dom-serializer "^2.0.0" + domelementtype "^2.3.0" + domhandler "^5.0.3" + dotignore@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/dotignore/-/dotignore-0.1.2.tgz#f942f2200d28c3a76fbdd6f0ee9f3257c8a2e905" @@ -1600,6 +1718,14 @@ emoji-regex@^8.0.0: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== +encoding-sniffer@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/encoding-sniffer/-/encoding-sniffer-0.2.0.tgz#799569d66d443babe82af18c9f403498365ef1d5" + integrity sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg== + dependencies: + iconv-lite "^0.6.3" + whatwg-encoding "^3.1.1" + end-of-stream@^1.0.0, end-of-stream@^1.1.0: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" @@ -1607,6 +1733,11 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0: dependencies: once "^1.4.0" +entities@^4.2.0, entities@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48" + integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw== + error-ex@^1.3.1: version "1.3.2" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" @@ -2039,6 +2170,11 @@ flatted@^3.2.9: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.1.tgz#21db470729a6734d4997002f439cb308987f567a" integrity sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw== +follow-redirects@^1.15.6: + version "1.15.9" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" + integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== + for-each@^0.3.3, for-each@~0.3.3: version "0.3.3" resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.3.tgz#69b447e88a0a5d32c3e7084f3f1710034b21376e" @@ -2046,6 +2182,15 @@ for-each@^0.3.3, for-each@~0.3.3: dependencies: is-callable "^1.1.3" +form-data@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.1.tgz#ba1076daaaa5bfd7e99c1a6cb02aa0a5cff90d48" + integrity sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + frac@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/frac/-/frac-1.1.2.tgz#3d74f7f6478c88a1b5020306d747dc6313c74d0b" @@ -2353,6 +2498,16 @@ html-escaper@^2.0.0: resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== +htmlparser2@^9.1.0: + version "9.1.0" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-9.1.0.tgz#cdb498d8a75a51f739b61d3f718136c369bc8c23" + integrity sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.3" + domutils "^3.1.0" + entities "^4.5.0" + http-cache-semantics@3.8.1: version "3.8.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" @@ -2363,6 +2518,13 @@ human-signals@^2.1.0: resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== +iconv-lite@0.6.3, iconv-lite@^0.6.3: + version "0.6.3" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" + integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== + dependencies: + safer-buffer ">= 2.1.2 < 3.0.0" + ieee754@^1.1.13: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -3276,11 +3438,18 @@ micromatch@^4.0.4: braces "^3.0.2" picomatch "^2.3.1" -mime-db@^1.28.0: +mime-db@1.52.0, mime-db@^1.28.0: version "1.52.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== +mime-types@^2.1.12: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" @@ -3400,6 +3569,13 @@ npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" +nth-check@^2.0.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.1.1.tgz#c9eab428effce36cd6b92c924bdb000ef1f1ed1d" + integrity sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w== + dependencies: + boolbase "^1.0.0" + object-assign@^4.0.1, object-assign@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" @@ -3581,6 +3757,28 @@ parse-json@^5.0.0, parse-json@^5.2.0: json-parse-even-better-errors "^2.3.0" lines-and-columns "^1.1.6" +parse5-htmlparser2-tree-adapter@^7.0.0: + version "7.1.0" + resolved "https://registry.yarnpkg.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz#b5a806548ed893a43e24ccb42fbb78069311e81b" + integrity sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g== + dependencies: + domhandler "^5.0.3" + parse5 "^7.0.0" + +parse5-parser-stream@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz#d7c20eadc37968d272e2c02660fff92dd27e60e1" + integrity sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow== + dependencies: + parse5 "^7.0.0" + +parse5@^7.0.0, parse5@^7.1.2: + version "7.2.1" + resolved "https://registry.yarnpkg.com/parse5/-/parse5-7.2.1.tgz#8928f55915e6125f430cc44309765bf17556a33a" + integrity sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ== + dependencies: + entities "^4.5.0" + path-exists@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" @@ -3697,6 +3895,11 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + pump@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" @@ -3907,6 +4110,11 @@ safe-regex-test@^1.0.3: es-errors "^1.3.0" is-regex "^1.1.4" +"safer-buffer@>= 2.1.2 < 3.0.0": + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== + sax@>=0.6.0: version "1.3.0" resolved "https://registry.yarnpkg.com/sax/-/sax-1.3.0.tgz#a5dbe77db3be05c9d1ee7785dbd3ea9de51593d0" @@ -4346,6 +4554,11 @@ tsconfig-paths@^3.15.0: minimist "^1.2.6" strip-bom "^3.0.0" +tslib@^2.6.2: + version "2.8.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" + integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== + type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" @@ -4455,6 +4668,11 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici@^6.19.5: + version "6.21.0" + resolved "https://registry.yarnpkg.com/undici/-/undici-6.21.0.tgz#4b3d3afaef984e07b48e7620c34ed8a285ed4cd4" + integrity sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw== + universalify@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.1.tgz#168efc2180964e6386d061e094df61afe239b18d" @@ -4527,6 +4745,18 @@ walker@^1.0.8: dependencies: makeerror "1.0.12" +whatwg-encoding@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz#d0f4ef769905d426e1688f3e34381a99b60b76e5" + integrity sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ== + dependencies: + iconv-lite "0.6.3" + +whatwg-mimetype@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz#bc1bf94a985dc50388d54a9258ac405c3ca2fc0a" + integrity sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg== + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"