From ec5f90db2fcc70eebb2fe029ec18c3949e30c64c Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Mon, 4 Sep 2023 20:27:15 +0100 Subject: [PATCH 1/6] Add STAC metadata integration --- .../components/datasets/dataset-list-item.tsx | 2 +- .../components/exploration/data-utils.ts | 54 +++++- .../hooks/use-stac-metadata-datasets.ts | 181 ++++++++++++++++++ app/scripts/components/exploration/index.tsx | 3 + .../components/exploration/types.d.ts.ts | 22 ++- 5 files changed, 255 insertions(+), 7 deletions(-) create mode 100644 app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts diff --git a/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx b/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx index 44ffa5075..ca83707e4 100644 --- a/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx +++ b/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx @@ -308,7 +308,7 @@ export function DatasetListItem(props: DatasetListItemProps) { /> )} - {isVisible && isPopoverVisible && dataPoint && ( + {isDatasetSucceeded && isVisible && isPopoverVisible && dataPoint && ( { const parentDataset = Object.values(datasets).find((dataset) => @@ -24,7 +36,7 @@ export function reconcileDatasets( ids: string[], datasetsList: DatasetLayer[], reconciledDatasets: TimelineDataset[] -) { +): TimelineDataset[] { return ids.map((id) => { const alreadyReconciled = reconciledDatasets.find((d) => d.data.id === id); @@ -34,6 +46,10 @@ export function reconcileDatasets( const dataset = datasetsList.find((d) => d.id === id); + if (!dataset) { + throw new Error(`Dataset [${id}] not found`); + } + return { status: TimelineDatasetStatus.IDLE, data: dataset, @@ -51,3 +67,37 @@ export function reconcileDatasets( }; }); } + +export function resolveLayerTemporalExtent( + datasetId: string, + datasetData: StacDatasetData +): Date[] { + const { domain, isPeriodic, timeDensity } = datasetData; + + if (!domain || domain.length === 0) { + throw new Error(`Invalid domain on dataset [${datasetId}]`); + } + + if (!isPeriodic) return domain.map((d) => utcString2userTzDate(d)); + + if (timeDensity === TimeDensity.YEAR) { + return eachYearOfInterval({ + start: utcString2userTzDate(domain[0]), + end: utcString2userTzDate(domain.last) + }); + } else if (timeDensity === TimeDensity.MONTH) { + return eachMonthOfInterval({ + start: utcString2userTzDate(domain[0]), + end: utcString2userTzDate(domain.last) + }); + } else if (timeDensity === TimeDensity.DAY) { + return eachDayOfInterval({ + start: utcString2userTzDate(domain[0]), + end: utcString2userTzDate(domain.last) + }); + } + + throw new Error( + `Invalid time density [${timeDensity}] on dataset [${datasetId}]` + ); +} diff --git a/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts b/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts new file mode 100644 index 000000000..08f0314e7 --- /dev/null +++ b/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts @@ -0,0 +1,181 @@ + +import { + useQueries, + UseQueryOptions, + UseQueryResult +} from '@tanstack/react-query'; +import axios from 'axios'; +import { useAtom } from 'jotai'; + +import { timelineDatasetsAtom } from '../atoms/atoms'; +import { + StacDatasetData, + TimelineDataset, + TimelineDatasetData, + TimelineDatasetStatus +} from '../types.d.ts'; +import { resolveLayerTemporalExtent } from '../data-utils'; + +import { useEffectPrevious } from '$utils/use-effect-previous'; + + +function didDataChange(curr: UseQueryResult, prev?: UseQueryResult) { + const currKey = `${curr.errorUpdatedAt}-${curr.dataUpdatedAt}`; + const prevKey = `${prev?.errorUpdatedAt}-${prev?.dataUpdatedAt}`; + + return prevKey !== currKey; +} + +/** + * Merges STAC metadata with local dataset, computing the domain. + * + * @param queryData react-query response with data from STAC request + * @param dataset Local dataset data. + * + * @returns Reconciled dataset with STAC data. + */ +function reconcileQueryDataWithDataset( + queryData: UseQueryResult, + dataset: TimelineDataset +): TimelineDataset { + try { + let base: TimelineDataset = { + ...dataset, + status: queryData.status as TimelineDatasetStatus, + error: queryData.error + }; + + if (queryData.status === TimelineDatasetStatus.SUCCEEDED) { + const domain = resolveLayerTemporalExtent(base.data.id, queryData.data); + + base = { + ...base, + data: { + ...base.data, + ...queryData.data, + domain + } + }; + } + + + return base; + } catch (error) { + const e = new Error('Error reconciling query data with dataset'); + // @ts-expect-error detail is not a property of Error + e.detail = error; + + return { + ...dataset, + status: TimelineDatasetStatus.ERRORED, + error: e + }; + } +} + +async function fetchStacDatasetById( + dataset: TimelineDatasetData +): Promise { + const { type, stacCol } = dataset; + + const { data } = await axios.get( + `${process.env.API_STAC_ENDPOINT}/collections/${stacCol}` + ); + + const commonTimeseriesParams = { + isPeriodic: data['dashboard:is_periodic'], + timeDensity: data['dashboard:time_density'] + }; + + if (type === 'vector') { + const featuresApiEndpoint = data.links.find( + (l) => l.rel === 'external' + ).href; + const { data: featuresApiData } = await axios.get(featuresApiEndpoint); + + return { + ...commonTimeseriesParams, + domain: featuresApiData.extent.temporal.interval[0] + }; + } else { + const domain = data.summaries + ? data.summaries.datetime + : data.extent.temporal.interval[0]; + + return { + ...commonTimeseriesParams, + domain + }; + } +} + +// Create a query object for react query. +function makeQueryObject( + dataset: TimelineDataset +): UseQueryOptions { + return { + queryKey: ['dataset', dataset.data.id], + queryFn: () => fetchStacDatasetById(dataset.data), + // This data will not be updated in the context of a browser session, so it is + // safe to set the staleTime to Infinity. As specified by react-query's + // "Important Defaults", cached data is considered stale which means that + // there would be a constant refetching. + staleTime: Infinity, + // Errors are always considered stale. If any layer errors, any refocus would + // cause a refetch. This is normally a good thing but since we have a refetch + // button, this is not needed. + refetchOnMount: false, + refetchOnReconnect: false, + refetchOnWindowFocus: false + }; +} + +/** + * Extends local dataset state with STAC metadata. + * Whenever a dataset is added to the timeline, this hook will fetch the STAC + * metadata for that dataset and add it to the dataset state atom. + */ +export function useStacMetadataOnDatasets() { + const [datasets, setDatasets] = useAtom(timelineDatasetsAtom); + + const datasetsQueryData = useQueries({ + queries: datasets.map((dataset) => makeQueryObject(dataset)) + }); + + useEffectPrevious< + [typeof datasetsQueryData, TimelineDataset[]] + >( + (prev) => { + const prevQueryData = prev[0]; + if (!prevQueryData) return; + + const { changed, data: updatedDatasets } = datasets.reduce<{ + changed: boolean; + data: TimelineDataset[]; + }>( + (acc, dataset, idx) => { + const curr = datasetsQueryData[idx]; + + if (didDataChange(curr, prevQueryData[idx])) { + // Changed + return { + changed: true, + data: [...acc.data, reconcileQueryDataWithDataset(curr, dataset)] + }; + } else { + return { + ...acc, + data: [...acc.data, dataset] + }; + } + }, + { changed: false, data: [] } + ); + + if (changed as boolean) { + setDatasets(updatedDatasets); + } + }, + [datasetsQueryData, datasets] + ); +} diff --git a/app/scripts/components/exploration/index.tsx b/app/scripts/components/exploration/index.tsx index e8c72785b..b8804b922 100644 --- a/app/scripts/components/exploration/index.tsx +++ b/app/scripts/components/exploration/index.tsx @@ -6,6 +6,7 @@ import { themeVal } from '@devseed-ui/theme-provider'; import { MockControls } from './datasets-mock'; import Timeline from './components/timeline/timeline'; import { DatasetSelectorModal } from './components/dataset-selector-modal'; +import { useStacMetadataOnDatasets } from './hooks/use-stac-metadata-datasets'; import { LayoutProps } from '$components/common/layout-root'; import PageHero from '$components/common/page-hero'; @@ -58,6 +59,8 @@ function Exploration() { const openModal = useCallback(() => setDatasetModalRevealed(true), []); const closeModal = useCallback(() => setDatasetModalRevealed(false), []); + useStacMetadataOnDatasets(); + return ( <> & { - date: Date; + date: Date; }; export interface TimelineDatasetAnalysis { @@ -27,9 +35,15 @@ export interface TimelineDatasetAnalysis { }; } +export interface TimelineDatasetData extends DatasetLayer { + isPeriodic: boolean; + timeDensity: TimeDensity; + domain: Date[]; +} + export interface TimelineDataset { status: TimelineDatasetStatus; - data: any; + data: TimelineDatasetData; error: any; settings: { // user defined settings like visibility, opacity From 38b08aaeb36b0835fee0b54aec0cecd9b398e1a1 Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Mon, 4 Sep 2023 20:29:53 +0100 Subject: [PATCH 2/6] Rename errors to better match react query --- .../components/exploration/atoms/hooks.ts | 2 +- .../components/datasets/dataset-list-item.tsx | 18 +++++++------- .../components/timeline/timeline.tsx | 2 +- .../components/exploration/datasets-mock.tsx | 24 +++++++++---------- .../hooks/use-stac-metadata-datasets.ts | 4 ++-- .../components/exploration/types.d.ts.ts | 4 ++-- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/app/scripts/components/exploration/atoms/hooks.ts b/app/scripts/components/exploration/atoms/hooks.ts index 202783af8..52aa7222b 100644 --- a/app/scripts/components/exploration/atoms/hooks.ts +++ b/app/scripts/components/exploration/atoms/hooks.ts @@ -20,7 +20,7 @@ export function useTimelineDatasetsDomain() { return useMemo(() => { const successDatasets = datasets.filter( - (d) => d.status === TimelineDatasetStatus.SUCCEEDED + (d) => d.status === TimelineDatasetStatus.SUCCESS ); if (!successDatasets.length) return undefined; diff --git a/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx b/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx index ca83707e4..0ddc4e461 100644 --- a/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx +++ b/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx @@ -187,16 +187,16 @@ export function DatasetListItem(props: DatasetListItemProps) { data: dataPoint }); - const isDatasetError = dataset.status === TimelineDatasetStatus.ERRORED; + const isDatasetError = dataset.status === TimelineDatasetStatus.ERROR; const isDatasetLoading = dataset.status === TimelineDatasetStatus.LOADING; - const isDatasetSucceeded = dataset.status === TimelineDatasetStatus.SUCCEEDED; + const isDatasetSuccess = dataset.status === TimelineDatasetStatus.SUCCESS; const isAnalysisAndError = - isAnalysis && dataset.analysis.status === TimelineDatasetStatus.ERRORED; + isAnalysis && dataset.analysis.status === TimelineDatasetStatus.ERROR; const isAnalysisAndLoading = isAnalysis && dataset.analysis.status === TimelineDatasetStatus.LOADING; - const isAnalysisAndSucceeded = - isAnalysis && dataset.analysis.status === TimelineDatasetStatus.SUCCEEDED; + const isAnalysisAndSuccess = + isAnalysis && dataset.analysis.status === TimelineDatasetStatus.SUCCESS; const datasetLegend = dataset.data.legend; @@ -270,7 +270,7 @@ export function DatasetListItem(props: DatasetListItemProps) { /> )} - {isDatasetSucceeded && ( + {isDatasetSuccess && ( <> {isAnalysisAndLoading && ( )} - {isAnalysisAndSucceeded && ( + {isAnalysisAndSuccess && ( )} - {isDatasetSucceeded && !isAnalysis && ( + {isDatasetSuccess && !isAnalysis && ( )} - {isDatasetSucceeded && isVisible && isPopoverVisible && dataPoint && ( + {isDatasetSuccess && isVisible && isPopoverVisible && dataPoint && ( d.status === TimelineDatasetStatus.SUCCEEDED + (d) => d.status === TimelineDatasetStatus.SUCCESS ); // When a loaded dataset is added from an empty state, compute the correct diff --git a/app/scripts/components/exploration/datasets-mock.tsx b/app/scripts/components/exploration/datasets-mock.tsx index e3a3bc637..461270d17 100644 --- a/app/scripts/components/exploration/datasets-mock.tsx +++ b/app/scripts/components/exploration/datasets-mock.tsx @@ -12,7 +12,7 @@ import { } from './types.d.ts'; const chartData = { - status: 'succeeded', + status: 'success', meta: { total: 9, loaded: 9 @@ -105,7 +105,7 @@ const chartData = { }; const chartData2 = { - status: 'succeeded', + status: 'success', meta: { total: 15, loaded: 15 @@ -368,7 +368,7 @@ function makeAnalysis( function makeDataset( data, - status = TimelineDatasetStatus.SUCCEEDED, + status = TimelineDatasetStatus.SUCCESS, settings: Record = {}, analysis = makeAnalysis({}, {}) ): TimelineDataset { @@ -445,10 +445,10 @@ export function MockControls() { toggleDataset( makeDataset( { - id: 'errored', + id: 'error', name: 'Error dataset' }, - TimelineDatasetStatus.ERRORED + TimelineDatasetStatus.ERROR ) ) ); @@ -479,12 +479,12 @@ export function MockControls() { toggleDataset( makeDataset( datasetSingle, - TimelineDatasetStatus.SUCCEEDED, + TimelineDatasetStatus.SUCCESS, {}, makeAnalysis( chartData2.data, chartData2.meta, - TimelineDatasetStatus.SUCCEEDED + TimelineDatasetStatus.SUCCESS ) ) ) @@ -500,12 +500,12 @@ export function MockControls() { toggleDataset( makeDataset( dataset2020, - TimelineDatasetStatus.SUCCEEDED, + TimelineDatasetStatus.SUCCESS, {}, makeAnalysis( chartData.data, chartData.meta, - TimelineDatasetStatus.SUCCEEDED + TimelineDatasetStatus.SUCCESS ) ) ) @@ -525,7 +525,7 @@ export function MockControls() { id: 'analysis-loading', name: 'Analysis loading' }, - TimelineDatasetStatus.SUCCEEDED, + TimelineDatasetStatus.SUCCESS, {}, makeAnalysis( {}, @@ -550,12 +550,12 @@ export function MockControls() { id: 'analysis-error', name: 'Analysis Error' }, - TimelineDatasetStatus.SUCCEEDED, + TimelineDatasetStatus.SUCCESS, {}, makeAnalysis( {}, { loaded: 34, total: 100 }, - TimelineDatasetStatus.ERRORED + TimelineDatasetStatus.ERROR ) ) ) diff --git a/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts b/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts index 08f0314e7..59a43547e 100644 --- a/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts +++ b/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts @@ -45,7 +45,7 @@ function reconcileQueryDataWithDataset( error: queryData.error }; - if (queryData.status === TimelineDatasetStatus.SUCCEEDED) { + if (queryData.status === TimelineDatasetStatus.SUCCESS) { const domain = resolveLayerTemporalExtent(base.data.id, queryData.data); base = { @@ -67,7 +67,7 @@ function reconcileQueryDataWithDataset( return { ...dataset, - status: TimelineDatasetStatus.ERRORED, + status: TimelineDatasetStatus.ERROR, error: e }; } diff --git a/app/scripts/components/exploration/types.d.ts.ts b/app/scripts/components/exploration/types.d.ts.ts index 856842229..421e0530f 100644 --- a/app/scripts/components/exploration/types.d.ts.ts +++ b/app/scripts/components/exploration/types.d.ts.ts @@ -9,8 +9,8 @@ export enum TimeDensity { export enum TimelineDatasetStatus { IDLE = 'idle', LOADING = 'loading', - SUCCEEDED = 'success', - ERRORED = 'error' + SUCCESS = 'success', + ERROR = 'error' } export interface StacDatasetData { From f976b2617d8dfda946eee04447c3101adb97293d Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Wed, 6 Sep 2023 11:16:08 +0100 Subject: [PATCH 3/6] Add type narrowing and discriminants --- .../components/exploration/atoms/hooks.ts | 11 ++- .../components/datasets/dataset-list-item.tsx | 10 +- .../components/exploration/data-utils.ts | 41 ++++---- .../components/exploration/datasets-mock.tsx | 8 +- .../hooks/use-stac-metadata-datasets.ts | 22 ++--- .../components/exploration/types.d.ts.ts | 95 +++++++++++++++++-- 6 files changed, 131 insertions(+), 56 deletions(-) diff --git a/app/scripts/components/exploration/atoms/hooks.ts b/app/scripts/components/exploration/atoms/hooks.ts index 52aa7222b..fb8cce62c 100644 --- a/app/scripts/components/exploration/atoms/hooks.ts +++ b/app/scripts/components/exploration/atoms/hooks.ts @@ -5,7 +5,11 @@ import { focusAtom } from 'jotai-optics'; import { add, max } from 'date-fns'; import { DAY_SIZE_MAX } from '../constants'; -import { TimelineDataset, TimelineDatasetStatus } from '../types.d.ts'; +import { + TimelineDataset, + TimelineDatasetStatus, + TimelineDatasetSuccess +} from '../types.d.ts'; import { timelineDatasetsAtom, timelineSizesAtom } from './atoms'; /** @@ -20,7 +24,8 @@ export function useTimelineDatasetsDomain() { return useMemo(() => { const successDatasets = datasets.filter( - (d) => d.status === TimelineDatasetStatus.SUCCESS + (d): d is TimelineDatasetSuccess => + d.status === TimelineDatasetStatus.SUCCESS ); if (!successDatasets.length) return undefined; @@ -28,7 +33,7 @@ export function useTimelineDatasetsDomain() { // is ordered and only look at first and last dates. const [start, end] = extent( successDatasets.flatMap((d) => - d.data.domain ? [d.data.domain[0], d.data.domain.last] : [] + d.data.domain.length ? [d.data.domain[0], d.data.domain.last] : [] ) ) as [Date, Date]; diff --git a/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx b/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx index 0ddc4e461..d7bf5721b 100644 --- a/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx +++ b/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx @@ -41,8 +41,8 @@ import { } from '$components/common/mapbox/layer-legend'; import { TimeDensity, - TimelineDataset, - TimelineDatasetStatus + TimelineDatasetStatus, + TimelineDatasetSuccess } from '$components/exploration/types.d.ts'; import { DATASET_TRACK_BLOCK_HEIGHT, @@ -174,7 +174,7 @@ export function DatasetListItem(props: DatasetListItemProps) { xScaled, containerWidth: width, layerX, - data: dataset.analysis.data.timeseries + data: dataset.analysis.data?.timeseries }); const { @@ -326,7 +326,7 @@ export function DatasetListItem(props: DatasetListItemProps) { interface DatasetTrackProps { width: number; xScaled: ScaleTime; - dataset: TimelineDataset; + dataset: TimelineDatasetSuccess; isVisible: boolean; } @@ -373,7 +373,7 @@ function DatasetTrack(props: DatasetTrackProps) { interface DatasetTrackBlockProps { xScaled: ScaleTime; date: Date; - dataset: TimelineDataset; + dataset: TimelineDatasetSuccess; isVisible: boolean; } diff --git a/app/scripts/components/exploration/data-utils.ts b/app/scripts/components/exploration/data-utils.ts index 28ac3af35..e151ef151 100644 --- a/app/scripts/components/exploration/data-utils.ts +++ b/app/scripts/components/exploration/data-utils.ts @@ -60,7 +60,7 @@ export function reconcileDatasets( }, analysis: { status: TimelineDatasetStatus.IDLE, - data: {}, + data: null, error: null, meta: {} } @@ -80,24 +80,25 @@ export function resolveLayerTemporalExtent( if (!isPeriodic) return domain.map((d) => utcString2userTzDate(d)); - if (timeDensity === TimeDensity.YEAR) { - return eachYearOfInterval({ - start: utcString2userTzDate(domain[0]), - end: utcString2userTzDate(domain.last) - }); - } else if (timeDensity === TimeDensity.MONTH) { - return eachMonthOfInterval({ - start: utcString2userTzDate(domain[0]), - end: utcString2userTzDate(domain.last) - }); - } else if (timeDensity === TimeDensity.DAY) { - return eachDayOfInterval({ - start: utcString2userTzDate(domain[0]), - end: utcString2userTzDate(domain.last) - }); + switch (timeDensity) { + case TimeDensity.YEAR: + return eachYearOfInterval({ + start: utcString2userTzDate(domain[0]), + end: utcString2userTzDate(domain.last) + }); + case TimeDensity.MONTH: + return eachMonthOfInterval({ + start: utcString2userTzDate(domain[0]), + end: utcString2userTzDate(domain.last) + }); + case TimeDensity.DAY: + return eachDayOfInterval({ + start: utcString2userTzDate(domain[0]), + end: utcString2userTzDate(domain.last) + }); + default: + throw new Error( + `Invalid time density [${timeDensity}] on dataset [${datasetId}]` + ); } - - throw new Error( - `Invalid time density [${timeDensity}] on dataset [${datasetId}]` - ); } diff --git a/app/scripts/components/exploration/datasets-mock.tsx b/app/scripts/components/exploration/datasets-mock.tsx index 461270d17..b8f28a53c 100644 --- a/app/scripts/components/exploration/datasets-mock.tsx +++ b/app/scripts/components/exploration/datasets-mock.tsx @@ -367,21 +367,21 @@ function makeAnalysis( } function makeDataset( - data, + data: any, status = TimelineDatasetStatus.SUCCESS, settings: Record = {}, analysis = makeAnalysis({}, {}) -): TimelineDataset { +) { return { status, data, - error: null, + error: status === TimelineDatasetStatus.ERROR ? new Error('Mock error') : null, settings: { ...settings, isVisible: settings.isVisible === undefined ? true : settings.isVisible }, analysis - }; + } as TimelineDataset; } function toggleDataset(dataset) { diff --git a/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts b/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts index 59a43547e..f709a48b6 100644 --- a/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts +++ b/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts @@ -1,4 +1,3 @@ - import { useQueries, UseQueryOptions, @@ -11,14 +10,12 @@ import { timelineDatasetsAtom } from '../atoms/atoms'; import { StacDatasetData, TimelineDataset, - TimelineDatasetData, TimelineDatasetStatus } from '../types.d.ts'; import { resolveLayerTemporalExtent } from '../data-utils'; import { useEffectPrevious } from '$utils/use-effect-previous'; - function didDataChange(curr: UseQueryResult, prev?: UseQueryResult) { const currKey = `${curr.errorUpdatedAt}-${curr.dataUpdatedAt}`; const prevKey = `${prev?.errorUpdatedAt}-${prev?.dataUpdatedAt}`; @@ -39,7 +36,7 @@ function reconcileQueryDataWithDataset( dataset: TimelineDataset ): TimelineDataset { try { - let base: TimelineDataset = { + let base = { ...dataset, status: queryData.status as TimelineDatasetStatus, error: queryData.error @@ -58,8 +55,7 @@ function reconcileQueryDataWithDataset( }; } - - return base; + return base as TimelineDataset; } catch (error) { const e = new Error('Error reconciling query data with dataset'); // @ts-expect-error detail is not a property of Error @@ -69,14 +65,14 @@ function reconcileQueryDataWithDataset( ...dataset, status: TimelineDatasetStatus.ERROR, error: e - }; + } as TimelineDataset; } } async function fetchStacDatasetById( - dataset: TimelineDatasetData + dataset: TimelineDataset ): Promise { - const { type, stacCol } = dataset; + const { type, stacCol } = dataset.data; const { data } = await axios.get( `${process.env.API_STAC_ENDPOINT}/collections/${stacCol}` @@ -115,7 +111,7 @@ function makeQueryObject( ): UseQueryOptions { return { queryKey: ['dataset', dataset.data.id], - queryFn: () => fetchStacDatasetById(dataset.data), + queryFn: () => fetchStacDatasetById(dataset), // This data will not be updated in the context of a browser session, so it is // safe to set the staleTime to Infinity. As specified by react-query's // "Important Defaults", cached data is considered stale which means that @@ -131,7 +127,7 @@ function makeQueryObject( } /** - * Extends local dataset state with STAC metadata. + * Extends local dataset state with STAC metadata. * Whenever a dataset is added to the timeline, this hook will fetch the STAC * metadata for that dataset and add it to the dataset state atom. */ @@ -142,9 +138,7 @@ export function useStacMetadataOnDatasets() { queries: datasets.map((dataset) => makeQueryObject(dataset)) }); - useEffectPrevious< - [typeof datasetsQueryData, TimelineDataset[]] - >( + useEffectPrevious<[typeof datasetsQueryData, TimelineDataset[]]>( (prev) => { const prevQueryData = prev[0]; if (!prevQueryData) return; diff --git a/app/scripts/components/exploration/types.d.ts.ts b/app/scripts/components/exploration/types.d.ts.ts index 421e0530f..60cb35f1e 100644 --- a/app/scripts/components/exploration/types.d.ts.ts +++ b/app/scripts/components/exploration/types.d.ts.ts @@ -1,4 +1,4 @@ -import { DatasetLayer } from "veda"; +import { DatasetLayer } from 'veda'; export enum TimeDensity { YEAR = 'year', @@ -20,20 +20,53 @@ export interface StacDatasetData { } export type AnalysisTimeseriesEntry = Record & { - date: Date; + date: Date; }; -export interface TimelineDatasetAnalysis { - status: TimelineDatasetStatus; - data: { - timeseries?: AnalysisTimeseriesEntry[]; +// TimelineDatasetAnalysis type discriminants +export interface TimelineDatasetAnalysisIdle { + status: TimelineDatasetStatus.IDLE; + data: null; + error: null; + meta: Record; +} +export interface TimelineDatasetAnalysisLoading { + status: TimelineDatasetStatus.LOADING; + data: null; + error: null; + meta: { + loaded?: number; + total?: number; }; - error: any; +} +export interface TimelineDatasetAnalysisError { + status: TimelineDatasetStatus.ERROR; + data: null; + error: unknown; meta: { loaded?: number; total?: number; }; } +export interface TimelineDatasetAnalysisSuccess { + status: TimelineDatasetStatus.SUCCESS; + data: { + timeseries: AnalysisTimeseriesEntry[]; + }; + error: null; + meta: { + loaded: number; + total: number; + }; +} + +export type TimelineDatasetAnalysis = + | TimelineDatasetAnalysisIdle + | TimelineDatasetAnalysisLoading + | TimelineDatasetAnalysisError + | TimelineDatasetAnalysisSuccess; + +// END TimelineDatasetAnalysis type discriminants export interface TimelineDatasetData extends DatasetLayer { isPeriodic: boolean; @@ -41,10 +74,44 @@ export interface TimelineDatasetData extends DatasetLayer { domain: Date[]; } -export interface TimelineDataset { - status: TimelineDatasetStatus; +// TimelineDataset type discriminants +export interface TimelineDatasetIdle { + status: TimelineDatasetStatus.IDLE; + data: DatasetLayer; + error: null; + settings: { + // user defined settings like visibility, opacity + isVisible?: boolean; + opacity?: number; + }; + analysis: TimelineDatasetAnalysisIdle; +} +export interface TimelineDatasetLoading { + status: TimelineDatasetStatus.LOADING; + data: DatasetLayer; + error: null; + settings: { + // user defined settings like visibility, opacity + isVisible?: boolean; + opacity?: number; + }; + analysis: TimelineDatasetAnalysisIdle; +} +export interface TimelineDatasetError { + status: TimelineDatasetStatus.ERROR; + data: DatasetLayer; + error: unknown; + settings: { + // user defined settings like visibility, opacity + isVisible?: boolean; + opacity?: number; + }; + analysis: TimelineDatasetAnalysisIdle; +} +export interface TimelineDatasetSuccess { + status: TimelineDatasetStatus.SUCCESS; data: TimelineDatasetData; - error: any; + error: null; settings: { // user defined settings like visibility, opacity isVisible?: boolean; @@ -53,6 +120,14 @@ export interface TimelineDataset { analysis: TimelineDatasetAnalysis; } +export type TimelineDataset = + | TimelineDatasetIdle + | TimelineDatasetLoading + | TimelineDatasetError + | TimelineDatasetSuccess; + +// END TimelineDataset type discriminants + export interface DateRange { start: Date; end: Date; From 8d5c880bc0628b3ef3e4bf4d5b99e4cbe361a133 Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Wed, 6 Sep 2023 12:33:32 +0100 Subject: [PATCH 4/6] Implement dataset metadata refetching --- .../datasets/dataset-list-item-status.tsx | 2 +- .../components/datasets/dataset-list-item.tsx | 18 +++++++++++++++--- .../hooks/use-stac-metadata-datasets.ts | 4 ++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/app/scripts/components/exploration/components/datasets/dataset-list-item-status.tsx b/app/scripts/components/exploration/components/datasets/dataset-list-item-status.tsx index 5e304e0d3..572dcbb0f 100644 --- a/app/scripts/components/exploration/components/datasets/dataset-list-item-status.tsx +++ b/app/scripts/components/exploration/components/datasets/dataset-list-item-status.tsx @@ -99,7 +99,7 @@ export function DatasetTrackError(props: {

{message}

{typeof onRetryClick === 'function' ? ( - ) : null} diff --git a/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx b/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx index d7bf5721b..a5805e264 100644 --- a/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx +++ b/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx @@ -1,4 +1,4 @@ -import React, { useMemo } from 'react'; +import React, { useCallback, useMemo } from 'react'; import { useAtomValue } from 'jotai'; import { Reorder, useDragControls } from 'framer-motion'; import styled, { useTheme } from 'styled-components'; @@ -13,6 +13,7 @@ import { startOfYear, areIntervalsOverlapping } from 'date-fns'; +import { useQueryClient } from '@tanstack/react-query'; import { ScaleTime } from 'd3'; import { CollecticonEye, @@ -157,6 +158,18 @@ export function DatasetListItem(props: DatasetListItemProps) { const [isVisible, setVisible] = useTimelineDatasetVisibility(datasetAtom); + const queryClient = useQueryClient(); + + const retryDatasetMetadata = useCallback(() => { + queryClient.invalidateQueries( + { + queryKey: ['dataset', datasetId], + exact: true + }, + { throwOnError: false } + ); + }, [queryClient, datasetId]); + const controls = useDragControls(); // Hook to handle the hover state of the dataset. Check the source file as to @@ -264,8 +277,7 @@ export function DatasetListItem(props: DatasetListItemProps) { { - /* eslint-disable-next-line no-console */ - console.log('Retry metadata loading'); + retryDatasetMetadata(); }} /> )} diff --git a/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts b/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts index f709a48b6..8e038e0ca 100644 --- a/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts +++ b/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts @@ -17,8 +17,8 @@ import { resolveLayerTemporalExtent } from '../data-utils'; import { useEffectPrevious } from '$utils/use-effect-previous'; function didDataChange(curr: UseQueryResult, prev?: UseQueryResult) { - const currKey = `${curr.errorUpdatedAt}-${curr.dataUpdatedAt}`; - const prevKey = `${prev?.errorUpdatedAt}-${prev?.dataUpdatedAt}`; + const currKey = `${curr.errorUpdatedAt}-${curr.dataUpdatedAt}-${curr.failureCount}`; + const prevKey = `${prev?.errorUpdatedAt}-${prev?.dataUpdatedAt}-${prev?.failureCount}`; return prevKey !== currKey; } From 4d341ed47156ef31a937112b99c4d0764fbf3382 Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Thu, 7 Sep 2023 15:01:19 +0100 Subject: [PATCH 5/6] Improve timeline dataset types --- .../components/exploration/types.d.ts.ts | 55 ++++++++----------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/app/scripts/components/exploration/types.d.ts.ts b/app/scripts/components/exploration/types.d.ts.ts index 60cb35f1e..bfc8ddc17 100644 --- a/app/scripts/components/exploration/types.d.ts.ts +++ b/app/scripts/components/exploration/types.d.ts.ts @@ -23,6 +23,11 @@ export type AnalysisTimeseriesEntry = Record & { date: Date; }; +interface AnalysisMeta { + loaded: number; + total: number; +} + // TimelineDatasetAnalysis type discriminants export interface TimelineDatasetAnalysisIdle { status: TimelineDatasetStatus.IDLE; @@ -34,19 +39,13 @@ export interface TimelineDatasetAnalysisLoading { status: TimelineDatasetStatus.LOADING; data: null; error: null; - meta: { - loaded?: number; - total?: number; - }; + meta: Partial } export interface TimelineDatasetAnalysisError { status: TimelineDatasetStatus.ERROR; data: null; error: unknown; - meta: { - loaded?: number; - total?: number; - }; + meta: Partial } export interface TimelineDatasetAnalysisSuccess { status: TimelineDatasetStatus.SUCCESS; @@ -54,10 +53,7 @@ export interface TimelineDatasetAnalysisSuccess { timeseries: AnalysisTimeseriesEntry[]; }; error: null; - meta: { - loaded: number; - total: number; - }; + meta: AnalysisMeta; } export type TimelineDatasetAnalysis = @@ -74,49 +70,44 @@ export interface TimelineDatasetData extends DatasetLayer { domain: Date[]; } +export interface TimelineDatasetSettings { + // Whether or not the layer should be shown on the map. + isVisible?: boolean; + // Opacity of the layer on the map. + opacity?: number; +} + // TimelineDataset type discriminants export interface TimelineDatasetIdle { status: TimelineDatasetStatus.IDLE; data: DatasetLayer; error: null; - settings: { - // user defined settings like visibility, opacity - isVisible?: boolean; - opacity?: number; - }; + // User controlled settings like visibility, opacity. + settings: TimelineDatasetSettings; analysis: TimelineDatasetAnalysisIdle; } export interface TimelineDatasetLoading { status: TimelineDatasetStatus.LOADING; data: DatasetLayer; error: null; - settings: { - // user defined settings like visibility, opacity - isVisible?: boolean; - opacity?: number; - }; + // User controlled settings like visibility, opacity. + settings: TimelineDatasetSettings; analysis: TimelineDatasetAnalysisIdle; } export interface TimelineDatasetError { status: TimelineDatasetStatus.ERROR; data: DatasetLayer; error: unknown; - settings: { - // user defined settings like visibility, opacity - isVisible?: boolean; - opacity?: number; - }; + // User controlled settings like visibility, opacity. + settings: TimelineDatasetSettings; analysis: TimelineDatasetAnalysisIdle; } export interface TimelineDatasetSuccess { status: TimelineDatasetStatus.SUCCESS; data: TimelineDatasetData; error: null; - settings: { - // user defined settings like visibility, opacity - isVisible?: boolean; - opacity?: number; - }; + // User controlled settings like visibility, opacity. + settings: TimelineDatasetSettings; analysis: TimelineDatasetAnalysis; } From 2bdfc834e02ab1be9c4f0f601c62bab811404505 Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Fri, 8 Sep 2023 17:07:04 +0100 Subject: [PATCH 6/6] Fix last domain date not being included --- .../components/exploration/atoms/hooks.ts | 18 ++++++++++++- .../hooks/use-stac-metadata-datasets.ts | 5 ++-- mock/datasets/sandbox.data.mdx | 26 +++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/app/scripts/components/exploration/atoms/hooks.ts b/app/scripts/components/exploration/atoms/hooks.ts index fb8cce62c..6de88a1bc 100644 --- a/app/scripts/components/exploration/atoms/hooks.ts +++ b/app/scripts/components/exploration/atoms/hooks.ts @@ -6,12 +6,23 @@ import { add, max } from 'date-fns'; import { DAY_SIZE_MAX } from '../constants'; import { + TimeDensity, TimelineDataset, TimelineDatasetStatus, TimelineDatasetSuccess } from '../types.d.ts'; import { timelineDatasetsAtom, timelineSizesAtom } from './atoms'; +function addDurationToDate(date, timeDensity: TimeDensity) { + const duration = { + [TimeDensity.YEAR]: { years: 1 }, + [TimeDensity.MONTH]: { months: 1 }, + [TimeDensity.DAY]: { days: 1 } + }[timeDensity]; + + return add(date, duration); +} + /** * Calculates the date domain of the datasets, if any are selected. * @returns Dataset date domain or undefined. @@ -33,7 +44,12 @@ export function useTimelineDatasetsDomain() { // is ordered and only look at first and last dates. const [start, end] = extent( successDatasets.flatMap((d) => - d.data.domain.length ? [d.data.domain[0], d.data.domain.last] : [] + d.data.domain.length + ? [ + d.data.domain[0], + addDurationToDate(d.data.domain.last, d.data.timeDensity) + ] + : [] ) ) as [Date, Date]; diff --git a/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts b/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts index 8e038e0ca..7aa6167ea 100644 --- a/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts +++ b/app/scripts/components/exploration/hooks/use-stac-metadata-datasets.ts @@ -9,6 +9,7 @@ import { useAtom } from 'jotai'; import { timelineDatasetsAtom } from '../atoms/atoms'; import { StacDatasetData, + TimeDensity, TimelineDataset, TimelineDatasetStatus } from '../types.d.ts'; @@ -79,8 +80,8 @@ async function fetchStacDatasetById( ); const commonTimeseriesParams = { - isPeriodic: data['dashboard:is_periodic'], - timeDensity: data['dashboard:time_density'] + isPeriodic: !!data['dashboard:is_periodic'], + timeDensity: data['dashboard:time_density'] || TimeDensity.DAY }; if (type === 'vector') { diff --git a/mock/datasets/sandbox.data.mdx b/mock/datasets/sandbox.data.mdx index 5138448e8..511567806 100644 --- a/mock/datasets/sandbox.data.mdx +++ b/mock/datasets/sandbox.data.mdx @@ -161,6 +161,32 @@ layers: label: "Out of season" - color: "#804115" label: "No data" + - id: facebook_population_density + stacCol: facebook_population_density + name: 'Facebook Population Density' + type: raster + description: 'Facebook high-resolution population density with a 30m² resolution' + zoomExtent: + - 0 + - 20 + sourceParams: + resampling: bilinear + bidx: 1 + colormap_name: ylorrd + rescale: + - 0 + - 69 + legend: + type: gradient + min: "0" + max: "69" + stops: + - "#ffffcc" + - "#fee187" + - "#feab49" + - "#fc5b2e" + - "#d41020" + - "#800026" related: - type: dataset id: no2