From d9e7d3c4adf78a2a2ddfa77e5199e42a08eacda5 Mon Sep 17 00:00:00 2001 From: Wille Marcel Date: Wed, 6 Nov 2024 16:58:44 +0700 Subject: [PATCH] Add AfriSAR campaign and handle kml/kmz files --- campaigns/AfriSAR/deployments.yaml | 25 +++++++++ task/package.json | 4 +- task/processAll.js | 4 +- task/{convertAll.js => processPlatform.js} | 16 +++++- task/src/convert-kml.js | 43 ++++++++++++++ task/src/download.js | 13 ++++- task/yarn.lock | 65 +++++++++++++++++++--- 7 files changed, 154 insertions(+), 16 deletions(-) create mode 100644 campaigns/AfriSAR/deployments.yaml rename task/{convertAll.js => processPlatform.js} (66%) create mode 100644 task/src/convert-kml.js diff --git a/campaigns/AfriSAR/deployments.yaml b/campaigns/AfriSAR/deployments.yaml new file mode 100644 index 0000000..ef3bee4 --- /dev/null +++ b/campaigns/AfriSAR/deployments.yaml @@ -0,0 +1,25 @@ +--- +deployments: +- name: AfriSAR-D1_2016 + platforms: + - name: B-200 + files: + - https://lvis.gsfc.nasa.gov/images/kmz/traj57438_LVIS.kmz + - https://lvis.gsfc.nasa.gov/images/kmz/traj57440_LVIS.kmz + - https://lvis.gsfc.nasa.gov/images/kmz/traj57441_LVIS.kmz + - https://lvis.gsfc.nasa.gov/images/kmz/traj57443_LVIS.kmz + - https://lvis.gsfc.nasa.gov/images/kmz/traj57449_LVIS.kmz + - https://lvis.gsfc.nasa.gov/images/kmz/traj57450_LVIS.kmz + - https://lvis.gsfc.nasa.gov/images/kmz/traj57451_LVIS.kmz + - https://lvis.gsfc.nasa.gov/images/kmz/traj57454_LVIS.kmz + - https://lvis.gsfc.nasa.gov/images/kmz/traj57455_LVIS.kmz + - name: G-III + files: + - https://uavsar.jpl.nasa.gov/flightplans/flt16008/PLAN_16008.kml + - https://uavsar.jpl.nasa.gov/flightplans/flt16009/PLAN_16009.kml + - https://uavsar.jpl.nasa.gov/flightplans/flt16010/PLAN_16010.kml + - https://uavsar.jpl.nasa.gov/flightplans/flt16011/PLAN_16011.kml + - https://uavsar.jpl.nasa.gov/flightplans/flt16012/PLAN_16012.kml + - https://uavsar.jpl.nasa.gov/flightplans/flt16013/PLAN_16013.kml + - https://uavsar.jpl.nasa.gov/flightplans/flt16014/PLAN_16014.kml + - https://uavsar.jpl.nasa.gov/flightplans/flt16015/PLAN_16015.kml diff --git a/task/package.json b/task/package.json index 463d479..8a295be 100644 --- a/task/package.json +++ b/task/package.json @@ -12,8 +12,8 @@ }, "dependencies": { "@mapbox/geojson-merge": "^1.1.1", + "@tmcw/togeojson": "^5.8.1", "@turf/distance": "^6.5.0", - "adm-zip": "^0.5.12", "csv2geojson": "^5.1.2", "d3-dsv": "1.0.1", "download": "8.0.0", @@ -24,8 +24,10 @@ "simplify-geojson": "^1.0.5", "slugify": "^1.6.6", "superstruct": "^1.0.4", + "unzipper": "^0.12.3", "xlsx": "^0.18.5", "xml2js": "^0.6.2", + "xmldom": "^0.6.0", "yaml": "2.3.4" }, "devDependencies": { diff --git a/task/processAll.js b/task/processAll.js index 505146e..d47414c 100644 --- a/task/processAll.js +++ b/task/processAll.js @@ -4,7 +4,7 @@ const fs = require('fs'); const { findDirectories, findFiles } = require('./src/find'); const { exportHeaders } = require('./src/headers'); const { makeCSV } = require('./makeCSV'); -const { convertAll } = require('./convertAll'); +const { makePlatformGeoJSON } = require('./processPlatform'); const { convert } = require('./convert'); const { mergeGeoJSONCollection } = require('./src/process'); @@ -14,7 +14,7 @@ const platforms = findDirectories(campaignPath, 2); platforms.forEach((p) => { exportHeaders(p); makeCSV(p); - convertAll(p); + makePlatformGeoJSON(p); }); // convert the static CSV file to GeoJSON diff --git a/task/convertAll.js b/task/processPlatform.js similarity index 66% rename from task/convertAll.js rename to task/processPlatform.js index 5d1065d..81e3629 100644 --- a/task/convertAll.js +++ b/task/processPlatform.js @@ -8,17 +8,27 @@ const { mergeGeoJSONCollection, } = require('./src/process'); const { getPlatformConfig } = require('./src/utils'); +const { kml2geojson } = require('./src/convert-kml'); -const convertAll = (dir) => { +const makePlatformGeoJSON = (dir) => { const properties = getPropertiesFromPath(dir); const platformConfig = getPlatformConfig(dir); const files = fs.readdirSync(dir); - const collection = files + let collection; + // convert CSV files to GeoJSON + collection = files .filter((f) => f.endsWith('.csv') && !f.endsWith('headers.csv')) .map((f) => path.join(dir, f)) .map((f) => convertToGeoJSON(f, properties, platformConfig.coords_divisor)); + // if the platform has + if (!collection.length && files.every((f) => f.endsWith('.kml'))) { + collection = files + .map((f) => path.join(dir, f)) + .map((f) => kml2geojson(f, properties)); + } + const resultFile = path.join( dir, slugify(`${properties.deployment}-${properties.platform_name}.geojson`) @@ -28,5 +38,5 @@ const convertAll = (dir) => { }; module.exports = { - convertAll, + makePlatformGeoJSON, }; diff --git a/task/src/convert-kml.js b/task/src/convert-kml.js new file mode 100644 index 0000000..e89c957 --- /dev/null +++ b/task/src/convert-kml.js @@ -0,0 +1,43 @@ +const toGeojson = require('@tmcw/togeojson'); +const fs = require('fs'); +const { DOMParser } = require('xmldom'); +const unzipper = require('unzipper'); +const simplify = require('simplify-geojson'); + +const kml2geojson = (filePath, properties) => { + const file = fs.readFileSync(filePath); + const kml = new DOMParser().parseFromString(file.toString()); + const geojson = toGeojson.kml(kml); + const features = geojson.features + .filter((f) => f.geometry.type === 'LineString') + .map((f) => ({ ...f, properties })); + return simplify({ ...geojson, features }, 0.001); +}; + +const kmz2kml = (filePath) => new Promise((resolve, reject) => { + fs.createReadStream(filePath) + .pipe(unzipper.Parse()) + .on('entry', (entry) => { + if (entry.path.indexOf('.kml') === -1) { + entry.autodrain(); + return; + } + let data = ''; + + entry.on('error', reject); + + entry.on('data', (chunk) => { + data += chunk; + }); + + entry.on('end', () => { + resolve(data); + }); + }) + .on('error', reject); +}); + +module.exports = { + kml2geojson, + kmz2kml, +}; diff --git a/task/src/download.js b/task/src/download.js index 616e0ba..3f1612b 100644 --- a/task/src/download.js +++ b/task/src/download.js @@ -1,8 +1,9 @@ const fs = require('fs'); const path = require('path'); -const AdmZip = require('adm-zip'); +const unzipper = require('unzipper'); const download = require('download'); const { getPlatformConfig, readCampaignYaml, urlHasFileExtension } = require('./utils'); +const { kmz2kml } = require('./convert-kml'); const replaceSlash = (str) => str.replaceAll('/', '-'); @@ -15,8 +16,8 @@ const downloadFile = async (url, dir) => { // if the file is a zip, decompress it if (url.endsWith('.zip')) { const filePath = path.join(dir, path.basename(url)); - const zip = new AdmZip(filePath); - zip.extractAllTo(dir); + const zip = await unzipper.Open.file(filePath); + await zip.extract({ path: dir }); fs.unlinkSync(filePath); } // The GRIP campaign has files without an extension and others with .dat that should be txt @@ -24,6 +25,12 @@ const downloadFile = async (url, dir) => { const filePath = path.join(dir, path.basename(url)); fs.renameSync(filePath, `${filePath}.txt`); } + if (url.endsWith('.kmz')) { + const filePath = path.join(dir, path.basename(url)); + const kml = await kmz2kml(filePath); + fs.writeFileSync(filePath.replace('.kmz', '.kml'), kml); + fs.unlinkSync(filePath); + } }; const downloadCampaign = (campaignPath) => { diff --git a/task/yarn.lock b/task/yarn.lock index 6fcf960..1f372f5 100644 --- a/task/yarn.lock +++ b/task/yarn.lock @@ -674,6 +674,11 @@ dependencies: "@sinonjs/commons" "^3.0.0" +"@tmcw/togeojson@^5.8.1": + version "5.8.1" + resolved "https://registry.yarnpkg.com/@tmcw/togeojson/-/togeojson-5.8.1.tgz#6cbcc9b1484ed28e71bdd4b5f96ceae540a4533f" + integrity sha512-2YNrbis3l5kS0XrYwiHEZcGwiRp0MJ5CvwGwtMWp2z2tsVlskeec2qgvKHnF0RCwI5GnjrrBOoKsWfndEnd3LA== + "@turf/distance@^6.5.0": version "6.5.0" resolved "https://registry.yarnpkg.com/@turf/distance/-/distance-6.5.0.tgz#21f04d5f86e864d54e2abde16f35c15b4f36149a" @@ -820,11 +825,6 @@ adler-32@~1.3.0: resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.3.1.tgz#1dbf0b36dda0012189a32b3679061932df1821e2" integrity sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A== -adm-zip@^0.5.12: - version "0.5.12" - resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.12.tgz#87786328e91d54b37358d8a50f954c4cd73ba60b" - integrity sha512-6TVU49mK6KZb4qG6xWaaM4C7sA/sgUMLy/JYMOzkcp3BvVLpW0fXDFQiIzAuxFCt/2+xD7fNIiPFAoLZPhVNLQ== - ajv@^6.12.4: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" @@ -1058,6 +1058,11 @@ bl@^1.0.0: readable-stream "^2.3.5" safe-buffer "^5.1.1" +bluebird@~3.7.2: + version "3.7.2" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" + integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== + brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" @@ -1568,6 +1573,13 @@ download@8.0.0: p-event "^2.1.0" pify "^4.0.1" +duplexer2@~0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1" + integrity sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA== + dependencies: + readable-stream "^2.0.2" + duplexer3@^0.1.4: version "0.1.5" resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.5.tgz#0b5e4d7bad5de8901ea4440624c8e1d20099217e" @@ -2052,6 +2064,15 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== +fs-extra@^11.2.0: + version "11.2.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.2.0.tgz#e70e17dfad64232287d01929399e0ea7c86b0e5b" + integrity sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -2237,7 +2258,7 @@ got@^8.3.1: url-parse-lax "^3.0.0" url-to-options "^1.0.1" -graceful-fs@^4.1.10, graceful-fs@^4.2.9: +graceful-fs@^4.1.10, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.2, graceful-fs@^4.2.9: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== @@ -3081,6 +3102,15 @@ json5@^2.2.3: resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== +jsonfile@^6.0.1: + version "6.1.0" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" + integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== + dependencies: + universalify "^2.0.0" + optionalDependencies: + graceful-fs "^4.1.6" + jsonparse@^1.2.0: version "1.3.1" resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" @@ -3728,7 +3758,7 @@ read-pkg@^5.2.0: parse-json "^5.0.0" type-fest "^0.6.0" -readable-stream@^2.0.0, readable-stream@^2.3.0, readable-stream@^2.3.5: +readable-stream@^2.0.0, readable-stream@^2.0.2, readable-stream@^2.3.0, readable-stream@^2.3.5: version "2.3.8" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b" integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== @@ -4425,6 +4455,22 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +universalify@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.1.tgz#168efc2180964e6386d061e094df61afe239b18d" + integrity sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw== + +unzipper@^0.12.3: + version "0.12.3" + resolved "https://registry.yarnpkg.com/unzipper/-/unzipper-0.12.3.tgz#31958f5eed7368ed8f57deae547e5a673e984f87" + integrity sha512-PZ8hTS+AqcGxsaQntl3IRBw65QrBI6lxzqDEL7IAo/XCEqRTKGfOX56Vea5TH9SZczRVxuzk1re04z/YjuYCJA== + dependencies: + bluebird "~3.7.2" + duplexer2 "~0.1.4" + fs-extra "^11.2.0" + graceful-fs "^4.2.2" + node-int64 "^0.4.0" + update-browserslist-db@^1.0.13: version "1.0.13" resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz#3c5e4f5c083661bd38ef64b6328c26ed6c8248c4" @@ -4568,6 +4614,11 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmldom@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/xmldom/-/xmldom-0.6.0.tgz#43a96ecb8beece991cef382c08397d82d4d0c46f" + integrity sha512-iAcin401y58LckRZ0TkI4k0VSM1Qg0KGSc3i8rU+xrxe19A/BN1zHyVSJY7uoutVlaTSzYyk/v5AmkewAP7jtg== + xtend@^4.0.0: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"