Skip to content

Commit

Permalink
feat: fetched meta.json for meta data and append onto config array
Browse files Browse the repository at this point in the history
  • Loading branch information
@varun2948
varun2948 committed Jan 23, 2024
1 parent 7f31db8 commit 3610c55
Show file tree
Hide file tree
Showing 2 changed files with 125 additions and 42 deletions.
154 changes: 112 additions & 42 deletions frontend/src/components/projectDetail/downloadOsmData.js
View file
Original file line number Diff line number Diff line change
@@ -1,37 +1,39 @@
import React, { useState } from 'react';
import React, { useEffect, useState } from 'react';
import PropTypes from 'prop-types';
import { RoadIcon, HomeIcon, WavesIcon, TaskIcon, DownloadIcon } from '../svgIcons';
import FileFormatCard from './fileFormatCard';
import Popup from 'reactjs-popup';
import { EXPORT_TOOL_S3_URL } from '../../config';
import messages from './messages';
import { FormattedMessage } from 'react-intl';
import formatBytes from '../../utils/formatBytes';
import { format } from 'date-fns';

export const TITLED_ICONS = [
{
Icon: RoadIcon,
title: 'roads',
title: 'Roads',
value: 'ROADS',
featuretype: ['lines'],
formats: ['geojson', 'shp', 'kml'],
},
{
Icon: HomeIcon,
title: 'buildings',
title: 'Buildings',
value: 'BUILDINGS',
featuretype: ['polygons'],
formats: ['geojson', 'shp', 'kml'],
},
{
Icon: WavesIcon,
title: 'waterways',
title: 'Waterways',
value: 'WATERWAYS',
featuretype: ['lines', 'polygons'],
formats: ['geojson', 'shp', 'kml'],
},
{
Icon: TaskIcon,
title: 'landuse',
title: 'Landuse',
value: 'LAND_USE',
featuretype: ['points', 'polygons'],
formats: ['geojson', 'shp', 'kml'],
Expand All @@ -49,6 +51,7 @@ export const DownloadOsmData = ({ projectMappingTypes, project }) => {
const [showPopup, setShowPopup] = useState(false);
const [isDownloadingState, setIsDownloadingState] = useState(null);
const [selectedCategoryFormat, setSelectedCategoryFormat] = useState(null);
const [mergedJSONData, setMergedJSONData] = useState(null);

const datasetConfig = {
dataset_prefix: `hotosm_project_${project.projectId}`,
Expand All @@ -63,13 +66,55 @@ export const DownloadOsmData = ({ projectMappingTypes, project }) => {
* @param {string} feature_type - The feature type of the ffile.
* @return {Promise<void>} Promise that resolves when the download is complete.
*/
const fetchMetaJSON = async () => {
const metaJSONResponse = await fetch(
`https://api-prod.raw-data.hotosm.org/v1/s3/get/${datasetConfig.dataset_folder}/${datasetConfig.dataset_prefix}/meta.json`,
);
// const metaJSON = null;
// const metaJSON = null;
const metaJSON = await metaJSONResponse.json();
// console.log(metaJSON, 'metaJSON');
const filteredMappingTypes = TITLED_ICONS?.filter((icon) =>
projectMappingTypes?.includes(icon.value),
);
const mergedData = filteredMappingTypes.map((category) => {
const dataset = metaJSON?.datasets?.find((dataset) => category.title in dataset) || {};
const addedMetaJSON = dataset[category.title] || { resources: [] };

const resources = addedMetaJSON.resources.reduce((mergedResources, resource) => {
const formatIndex = category.formats.indexOf(resource.format);
if (formatIndex !== -1) {
const featureType = resource.name.includes('_polygons') ? 'polygons' : 'lines';
if (category.featuretype.includes(featureType)) {
if (!mergedResources[featureType]) {
mergedResources[featureType] = {};
}

mergedResources[featureType][category.formats[formatIndex]] = {
name: resource.name,
url: resource.url,
description: resource.description,
size: resource.size,
last_modifed: resource.last_modifed,
};
}
}
return mergedResources;
}, {});

return { ...category, resources };
});
setMergedJSONData(mergedData);
console.log(mergedData);
};

const downloadS3File = async (title, fileFormat, feature_type) => {
// Create the base URL for the S3 file
const downloadUrl = `${EXPORT_TOOL_S3_URL}/${datasetConfig.dataset_folder}/${
datasetConfig.dataset_prefix
}/${title}/${feature_type}/${
}/${title.toLowerCase()}/${feature_type}/${
datasetConfig.dataset_prefix
}_${title}_${feature_type}_${fileFormat.toLowerCase()}.zip`;
}_${title.toLowerCase()}_${feature_type}_${fileFormat.toLowerCase()}.zip`;

// Set the state to indicate that the file download is in progress
setIsDownloadingState({ title: title, fileFormat: fileFormat, isDownloading: true });
Expand Down Expand Up @@ -102,10 +147,10 @@ export const DownloadOsmData = ({ projectMappingTypes, project }) => {
console.error('Error:', error.message);
}
};
const filteredMappingTypes = TITLED_ICONS?.filter((icon) =>
projectMappingTypes?.includes(icon.value),
);

useEffect(() => {
fetchMetaJSON();
}, []);
return (
<div className="mb5 w-100 pb5 ph4 flex flex-wrap">
<Popup modal open={showPopup} closeOnDocumentClick nested onClose={() => setShowPopup(false)}>
Expand Down Expand Up @@ -134,13 +179,13 @@ export const DownloadOsmData = ({ projectMappingTypes, project }) => {
</div>
)}
</Popup>
{filteredMappingTypes.map((type) => {
{mergedJSONData?.map((type) => {
const loadingState = isDownloadingState?.isDownloading;
return (
<div
className="osm-card bg-white pa3 mr4 mt4 w-auto-m flex flex-wrap items-center "
style={{
width: '560px',
width: '870px',
gap: '16px',
}}
key={type.title}
Expand All @@ -161,8 +206,8 @@ export const DownloadOsmData = ({ projectMappingTypes, project }) => {
</div>
<div className="flex-column">
<div
className="file-list flex barlow-condensed f3"
style={{ display: 'flex', gap: '12px' }}
className="file-list flex barlow-condensed f3 "
style={{ display: 'flex', gap: '12px', width: 'fit-content' }}
>
<p className="fw5 ttc">{type.title}</p>
<FileFormatCard
Expand All @@ -185,34 +230,59 @@ export const DownloadOsmData = ({ projectMappingTypes, project }) => {
{selectedCategoryFormat &&
selectedCategoryFormat.title === type.title &&
type?.featuretype?.map((typ) => (
<span
key={`${typ}_${selectedCategoryFormat.title}`}
onClick={() =>
downloadS3File(
selectedCategoryFormat.title,
selectedCategoryFormat.format,
typ,
)
}
onKeyUp={() =>
downloadS3File(
selectedCategoryFormat.title,
selectedCategoryFormat.format,
typ,
)
}
style={
loadingState
? { cursor: 'not-allowed', pointerEvents: 'none', gap: '10px' }
: { cursor: 'pointer', gap: '10px' }
}
className="flex flex-row items-center pointer link hover-red color-inherit categorycard"
>
<DownloadIcon style={{ height: '28px' }} color="#D73F3F" />
<p className="ttc">
{typ} {selectedCategoryFormat.format}
</p>
</span>
<>
<span
key={`${typ}_${selectedCategoryFormat.title}`}
onClick={() =>
downloadS3File(
selectedCategoryFormat.title,
selectedCategoryFormat.format,
typ,
)
}
onKeyUp={() =>
downloadS3File(
selectedCategoryFormat.title,
selectedCategoryFormat.format,
typ,
)
}
style={
loadingState
? { cursor: 'not-allowed', pointerEvents: 'none', gap: '10px' }
: { cursor: 'pointer', gap: '10px' }
}
className="flex flex-row items-center pointer link hover-red color-inherit categorycard"
>
<div className="flex flex-column">
<div className="flex flex-row items-center ">
<DownloadIcon style={{ height: '28px' }} color="#D73F3F" />
<p className="ttc ml2">
{typ} {selectedCategoryFormat.format}
<span className="ml1 gray f7">
(
{formatBytes(
type.resources[typ][selectedCategoryFormat.format].size,
)}
)
</span>
</p>
</div>
<p className="gray f7" style={{ margin: 0 }}>
Last Generated:{' '}
{type.resources[typ][selectedCategoryFormat.format].last_modifed
? format(
new Date(
type.resources[typ][selectedCategoryFormat.format].last_modifed,
),
'MM/dd/yyyy HH:mm:ss z ',
{ timeZone: 'UTC' },
)
: '-'}
</p>
</div>
</span>
</>
))}
</div>
</div>
Expand Down
13 changes: 13 additions & 0 deletions frontend/src/utils/formatBytes.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
function formatBytes(bytes, decimals = 2) {
if (!+bytes) return '0 Bytes';

const k = 1024;
const dm = decimals < 0 ? 0 : decimals;
const sizes = ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];

const i = Math.floor(Math.log(bytes) / Math.log(k));

return `${parseFloat((bytes / Math.pow(k, i)).toFixed(dm))} ${sizes[i]}`;
}

export default formatBytes;

0 comments on commit 3610c55

Please sign in to comment.