diff --git a/frontend/src/components/projectDetail/downloadOsmData.js b/frontend/src/components/projectDetail/downloadOsmData.js index 05bf091abf..6abad84cd3 100644 --- a/frontend/src/components/projectDetail/downloadOsmData.js +++ b/frontend/src/components/projectDetail/downloadOsmData.js @@ -7,35 +7,34 @@ import { EXPORT_TOOL_S3_URL } from '../../config'; import messages from './messages'; import { FormattedMessage } from 'react-intl'; import formatBytes from '../../utils/formatBytes'; -import { format } from 'date-fns'; export const TITLED_ICONS = [ { Icon: RoadIcon, - title: 'Roads', + title: 'roads', value: 'ROADS', - featuretype: ['lines'], + featuretype: [{ type: 'lines' }], formats: ['geojson', 'shp', 'kml'], }, { Icon: HomeIcon, - title: 'Buildings', + title: 'buildings', value: 'BUILDINGS', - featuretype: ['polygons'], + featuretype: [{ type: 'polygons' }], formats: ['geojson', 'shp', 'kml'], }, { Icon: WavesIcon, - title: 'Waterways', + title: 'waterways', value: 'WATERWAYS', - featuretype: ['lines', 'polygons'], + featuretype: [{ type: 'polygons' }, { type: 'lines' }], formats: ['geojson', 'shp', 'kml'], }, { Icon: TaskIcon, - title: 'Landuse', + title: 'landuse', value: 'LAND_USE', - featuretype: ['points', 'polygons'], + featuretype: [{ type: 'points' }, { type: 'polygons' }], formats: ['geojson', 'shp', 'kml'], }, ]; @@ -48,10 +47,10 @@ export const TITLED_ICONS = [ */ export const DownloadOsmData = ({ projectMappingTypes, project }) => { + const [downloadDataList, setDownloadDataList] = useState(TITLED_ICONS); const [showPopup, setShowPopup] = useState(false); const [isDownloadingState, setIsDownloadingState] = useState(null); const [selectedCategoryFormat, setSelectedCategoryFormat] = useState(null); - const [mergedJSONData, setMergedJSONData] = useState(null); const datasetConfig = { dataset_prefix: `hotosm_project_${project.projectId}`, @@ -66,55 +65,13 @@ export const DownloadOsmData = ({ projectMappingTypes, project }) => { * @param {string} feature_type - The feature type of the ffile. * @return {Promise} Promise that resolves when the download is complete. */ - const fetchMetaJSON = async () => { - const metaJSONResponse = await fetch( - `https://api-prod.raw-data.hotosm.org/v1/s3/get/${datasetConfig.dataset_folder}/${datasetConfig.dataset_prefix}/meta.json`, - ); - // const metaJSON = null; - // const metaJSON = null; - const metaJSON = await metaJSONResponse.json(); - // console.log(metaJSON, 'metaJSON'); - const filteredMappingTypes = TITLED_ICONS?.filter((icon) => - projectMappingTypes?.includes(icon.value), - ); - const mergedData = filteredMappingTypes.map((category) => { - const dataset = metaJSON?.datasets?.find((dataset) => category.title in dataset) || {}; - const addedMetaJSON = dataset[category.title] || { resources: [] }; - - const resources = addedMetaJSON.resources.reduce((mergedResources, resource) => { - const formatIndex = category.formats.indexOf(resource.format); - if (formatIndex !== -1) { - const featureType = resource.name.includes('_polygons') ? 'polygons' : 'lines'; - if (category.featuretype.includes(featureType)) { - if (!mergedResources[featureType]) { - mergedResources[featureType] = {}; - } - - mergedResources[featureType][category.formats[formatIndex]] = { - name: resource.name, - url: resource.url, - description: resource.description, - size: resource.size, - last_modifed: resource.last_modifed, - }; - } - } - return mergedResources; - }, {}); - - return { ...category, resources }; - }); - setMergedJSONData(mergedData); - console.log(mergedData); - }; - const downloadS3File = async (title, fileFormat, feature_type) => { // Create the base URL for the S3 file const downloadUrl = `${EXPORT_TOOL_S3_URL}/${datasetConfig.dataset_folder}/${ datasetConfig.dataset_prefix - }/${title.toLowerCase()}/${feature_type}/${ + }/${title}/${feature_type}/${ datasetConfig.dataset_prefix - }_${title.toLowerCase()}_${feature_type}_${fileFormat.toLowerCase()}.zip`; + }_${title}_${feature_type}_${fileFormat.toLowerCase()}.zip`; // Set the state to indicate that the file download is in progress setIsDownloadingState({ title: title, fileFormat: fileFormat, isDownloading: true }); @@ -147,10 +104,95 @@ export const DownloadOsmData = ({ projectMappingTypes, project }) => { console.error('Error:', error.message); } }; + useEffect(() => { + const filteredMappingTypes = downloadDataList?.filter((icon) => + projectMappingTypes?.includes(icon.value), + ); + setDownloadDataList(filteredMappingTypes); + }, [projectMappingTypes]); useEffect(() => { - fetchMetaJSON(); - }, []); + if (!selectedCategoryFormat) return null; + async function fetchData(url) { + const response = await fetch(url, { method: 'HEAD' }); + const data = await response; + return { + size: data.headers.get('Content-Length'), + lastmod: data.headers.get('Last-Modified'), + }; + } + + const multipleHeadCallForFormat = async () => { + setIsDownloadingState({ + title: selectedCategoryFormat.title, + fileFormat: selectedCategoryFormat.format, + isDownloading: true, + }); + + const filterMappingCategory = downloadDataList.find( + (type) => type.title === selectedCategoryFormat.title, + ); + async function fetchAndMap(urls) { + const results = await Promise.all( + urls.map(async (url) => { + const data = await fetchData(url); + return data; + }), + ); + + return results; + } + const multipleUrl = filterMappingCategory.featuretype.map((type) => { + return `${EXPORT_TOOL_S3_URL}/${datasetConfig.dataset_folder}/${ + datasetConfig.dataset_prefix + }/${selectedCategoryFormat.title}/${type.type}/${datasetConfig.dataset_prefix}_${ + selectedCategoryFormat.title + }_${type.type}_${selectedCategoryFormat.format.toLowerCase()}.zip`; + }); + fetchAndMap(multipleUrl) + .then((results) => { + var mergedArray = filterMappingCategory.featuretype.map((feature, index) => ({ + ...feature, + ...results[index], + })); + const mergedListData = downloadDataList.map((type) => { + if (type.title === selectedCategoryFormat.title) { + return { + ...type, + featuretype: mergedArray, + }; + } + return type; + }); + setDownloadDataList(mergedListData); + setIsDownloadingState({ + title: selectedCategoryFormat.title, + fileFormat: selectedCategoryFormat.fileFormat, + isDownloading: false, + }); + }) + + .catch((error) => { + console.error(error); + setIsDownloadingState({ + title: selectedCategoryFormat.title, + fileFormat: selectedCategoryFormat.fileFormat, + isDownloading: false, + }); + }) + .finally(() => { + setIsDownloadingState({ + title: selectedCategoryFormat.title, + fileFormat: selectedCategoryFormat.fileFormat, + isDownloading: false, + }); + }); + }; + multipleHeadCallForFormat(); + + console.log('test'); + }, [selectedCategoryFormat]); + return (
setShowPopup(false)}> @@ -179,13 +221,13 @@ export const DownloadOsmData = ({ projectMappingTypes, project }) => {
)} - {mergedJSONData?.map((type) => { + {downloadDataList.map((type) => { const loadingState = isDownloadingState?.isDownloading; return (
{

{type.title}

{ {selectedCategoryFormat && selectedCategoryFormat.title === type.title && type?.featuretype?.map((typ) => ( - <> - - downloadS3File( - selectedCategoryFormat.title, - selectedCategoryFormat.format, - typ, - ) - } - onKeyUp={() => - downloadS3File( - selectedCategoryFormat.title, - selectedCategoryFormat.format, - typ, - ) - } - style={ - loadingState - ? { cursor: 'not-allowed', pointerEvents: 'none', gap: '10px' } - : { cursor: 'pointer', gap: '10px' } - } - className="flex flex-row items-center pointer link hover-red color-inherit categorycard" - > -
-
- -

- {typ} {selectedCategoryFormat.format} - - ( - {formatBytes( - type.resources[typ][selectedCategoryFormat.format].size, - )} - ) - -

-
-

- Last Generated:{' '} - {type.resources[typ][selectedCategoryFormat.format].last_modifed - ? format( - new Date( - type.resources[typ][selectedCategoryFormat.format].last_modifed, - ), - 'MM/dd/yyyy HH:mm:ss z ', - { timeZone: 'UTC' }, - ) - : '-'} -

-
-
- + + downloadS3File( + selectedCategoryFormat.title, + selectedCategoryFormat.format, + typ.type, + ) + } + onKeyUp={() => + downloadS3File( + selectedCategoryFormat.title, + selectedCategoryFormat.format, + typ.type, + ) + } + style={ + loadingState + ? { cursor: 'not-allowed', pointerEvents: 'none', gap: '10px' } + : { cursor: 'pointer', gap: '10px' } + } + className="flex flex-row items-center pointer link hover-red color-inherit categorycard" + > +
+

+ + + {typ.type} {selectedCategoryFormat.format} + + {`(${ + typ.size ? formatBytes(typ.size) : '-' + })`} +

+ {`Last Generated: ${ + typ.lastmod ? typ.lastmod : '-' + })`} +
+
))}