From a7f2fb1a9924e351cc66b0e3e5f20c1e3fd5fc13 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Wed, 13 Mar 2024 17:57:52 -0500 Subject: [PATCH 1/4] Add support for arcgis imageserver datasets visualization --- .../common/map/style-generators/hooks.ts | 47 ++++++ .../common/mapbox/layers/arc-timeseries.tsx | 158 ++++++++++++++++++ .../components/common/mapbox/layers/utils.ts | 6 + parcel-resolver-veda/index.d.ts | 2 +- 4 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 app/scripts/components/common/mapbox/layers/arc-timeseries.tsx diff --git a/app/scripts/components/common/map/style-generators/hooks.ts b/app/scripts/components/common/map/style-generators/hooks.ts index 5fbbe1830..22ba150a9 100644 --- a/app/scripts/components/common/map/style-generators/hooks.ts +++ b/app/scripts/components/common/map/style-generators/hooks.ts @@ -13,6 +13,17 @@ interface ZarrResponseData { } } } +interface Link { + href: string, + rel: string, + type: string, + title: string, + "wms:layers": Array, + "wms:styles": Array +} +interface ArcResponseData { + links: Array +} interface CMRResponseData { features: { assets: { @@ -110,4 +121,40 @@ export function useCMR({ id, stacCol, stacApiEndpointToUse, date, assetUrlReplac return assetUrl; +} + +export function useArc({ id, stacCol, stacApiEndpointToUse, date, onStatusChange }){ + const [wmsUrl, setWmsUrl] = useState(''); + + useEffect(() => { + const controller = new AbortController(); + + async function load() { + try { + onStatusChange?.({ status: S_LOADING, id }); + const data:ArcResponseData = await requestQuickCache({ + url: `${stacApiEndpointToUse}/collections/${stacCol}`, + method: 'GET', + controller + }); + + setWmsUrl(data.links[0].href); + onStatusChange?.({ status: S_SUCCEEDED, id }); + } catch (error) { + if (!controller.signal.aborted) { + setWmsUrl(''); + onStatusChange?.({ status: S_FAILED, id }); + } + return; + } + } + + load(); + + return () => { + controller.abort(); + }; + }, [id, stacCol, stacApiEndpointToUse, date, onStatusChange]); + + return wmsUrl; } \ No newline at end of file diff --git a/app/scripts/components/common/mapbox/layers/arc-timeseries.tsx b/app/scripts/components/common/mapbox/layers/arc-timeseries.tsx new file mode 100644 index 000000000..7ee5a6ce5 --- /dev/null +++ b/app/scripts/components/common/mapbox/layers/arc-timeseries.tsx @@ -0,0 +1,158 @@ +import React, { useEffect, useMemo } from 'react'; +import qs from 'qs'; +import { RasterSource, RasterLayer } from 'mapbox-gl'; + +import { useMapStyle } from './styles'; +import { useArc } from '$components/common/map/style-generators/hooks'; + +import { ActionStatus } from '$utils/status'; + +export interface MapLayerArcTimeseriesProps { + id: string; + stacCol: string; + date?: Date; + sourceParams?: Record; + stacApiEndpoint?: string; + tileApiEndpoint?: string; + zoomExtent?: number[]; + onStatusChange?: (result: { status: ActionStatus; id: string }) => void; + isHidden?: boolean; + idSuffix?: string; +} + +interface ArcPaintLayerProps { + id: string; + date?: Date; + sourceParams?: Record; + tileApiEndpoint?: string; + zoomExtent?: number[]; + isHidden?: boolean; + idSuffix?: string; + wmsUrl: string; +} + +export function ArcPaintLayer(props: ArcPaintLayerProps) { + const { + id, + tileApiEndpoint, + date, + sourceParams, + zoomExtent, + isHidden, + wmsUrl, + idSuffix = '' + } = props; + + const { updateStyle } = useMapStyle(); + + const [minZoom] = zoomExtent ?? [0, 20]; + + const generatorId = 'arc-timeseries' + idSuffix; + + // Generate Mapbox GL layers and sources for raster timeseries + // + const haveSourceParamsChanged = useMemo( + () => JSON.stringify(sourceParams), + [sourceParams] + ); + + useEffect( + () => { + if (!wmsUrl) return; + //https://arcgis.asdc.larc.nasa.gov/server/services/POWER/power_901_annual_meterology_utc/ImageServer/WMSServer + // ?bbox={bbox-epsg-3857}&format=image/png&service=WMS&version=1.1.1&request=GetMap&srs=EPSG:3857&transparent=true&width=256&height=256&LAYERS=PS&DIM_StdTime=1981-12-31T00:00:00Z" + + // TODO: investigate For some reason the request being made is 6 hours ahead? Something to do with UTC <-> local conversion? + date?.setHours(date.getHours() - 6) + + const tileParams = qs.stringify({ + format: 'image/png', + service: "WMS", + version: "1.1.1", + request: "GetMap", + srs: "EPSG:3857", + transparent: "true", // TODO: get from sourceparams maybe + width: "256", + height: "256", + DIM_StdTime: `${date?.toISOString().slice(0, -5)}Z`, // TODO: better date conversion + ...sourceParams + }); + + const arcSource: RasterSource = { + type: 'raster', + tiles: [`${wmsUrl}?${tileParams}&bbox={bbox-epsg-3857}`] + }; + + const arcLayer: RasterLayer = { + id: id, + type: 'raster', + source: id, + layout: { + visibility: isHidden ? 'none' : 'visible' + }, + paint: { + 'raster-opacity': Number(!isHidden), + 'raster-opacity-transition': { + duration: 320 + } + }, + minzoom: minZoom, + metadata: { + layerOrderPosition: 'raster' + } + }; + + const sources = { + [id]: arcSource + }; + const layers = [arcLayer]; + + updateStyle({ + generatorId, + sources, + layers + }); + }, + // sourceParams not included, but using a stringified version of it to detect changes (haveSourceParamsChanged) + [ + updateStyle, + id, + date, + wmsUrl, + minZoom, + haveSourceParamsChanged, + isHidden, + generatorId, + tileApiEndpoint + ] + ); + + // + // Cleanup layers on unmount. + // + useEffect(() => { + return () => { + updateStyle({ + generatorId, + sources: {}, + layers: [] + }); + }; + }, [updateStyle, generatorId]); + + return null; +} + +export function MapLayerArcTimeseries(props:MapLayerArcTimeseriesProps) { + const { + id, + stacCol, + stacApiEndpoint, + date, + onStatusChange, + } = props; + + const stacApiEndpointToUse = stacApiEndpoint?? process.env.API_STAC_ENDPOINT; + const wmsUrl = useArc({id, stacCol, stacApiEndpointToUse, date, onStatusChange}); + return ; +} diff --git a/app/scripts/components/common/mapbox/layers/utils.ts b/app/scripts/components/common/mapbox/layers/utils.ts index b8a310dc8..cf5ded817 100644 --- a/app/scripts/components/common/mapbox/layers/utils.ts +++ b/app/scripts/components/common/mapbox/layers/utils.ts @@ -37,6 +37,10 @@ import { MapLayerCMRTimeseries, MapLayerCMRTimeseriesProps } from './cmr-timeseries'; +import { + MapLayerArcTimeseries, + MapLayerArcTimeseriesProps +} from './arc-timeseries'; import { userTzDate2utcString, utcString2userTzDate } from '$utils/date'; import { AsyncDatasetLayer } from '$context/layer-data'; @@ -51,12 +55,14 @@ export const getLayerComponent = ( | MapLayerVectorTimeseriesProps | MapLayerZarrTimeseriesProps | MapLayerCMRTimeseriesProps + | MapLayerArcTimeseriesProps > | null => { if (isTimeseries) { if (layerType === 'raster') return MapLayerRasterTimeseries; if (layerType === 'vector') return MapLayerVectorTimeseries; if (layerType === 'zarr') return MapLayerZarrTimeseries; if (layerType === 'cmr') return MapLayerCMRTimeseries; + if (layerType === 'arc') return MapLayerArcTimeseries; } return null; diff --git a/parcel-resolver-veda/index.d.ts b/parcel-resolver-veda/index.d.ts index c242788b1..4c305c686 100644 --- a/parcel-resolver-veda/index.d.ts +++ b/parcel-resolver-veda/index.d.ts @@ -7,7 +7,7 @@ declare module 'veda' { // /////////////////////////////////////////////////////////////////////////// // Datasets // // /////////////////////////////////////////////////////////////////////////// - type DatasetLayerType = 'raster' | 'vector' | 'zarr'| 'cmr'; + type DatasetLayerType = 'raster' | 'vector' | 'zarr'| 'cmr' | 'arc'; // // Dataset Layers From cc6bae3a198554f8b5a9de77066c434fd2602194 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Wed, 13 Mar 2024 17:59:17 -0500 Subject: [PATCH 2/4] Add analysis support for arcgis imageserver dataset --- .../analysis/results/timeseries-data.ts | 249 +++++++++++------- 1 file changed, 148 insertions(+), 101 deletions(-) diff --git a/app/scripts/components/analysis/results/timeseries-data.ts b/app/scripts/components/analysis/results/timeseries-data.ts index 2a5ea4396..c46373624 100644 --- a/app/scripts/components/analysis/results/timeseries-data.ts +++ b/app/scripts/components/analysis/results/timeseries-data.ts @@ -8,27 +8,30 @@ import { getFilterPayload, combineFeatureCollection } from '../utils'; import EventEmitter from './mini-events'; import { ConcurrencyManager, ConcurrencyManagerInstance } from './concurrency'; import { TimeDensity } from '$context/layer-data'; +import { userTzDate2utcString } from '$utils/date'; + export const TIMESERIES_DATA_BASE_ID = 'analysis'; +// ArcGIS ImageServer doesn't give back all these values export interface TimeseriesDataUnit { date: string; - min: number; - max: number; + min?: number; + max?: number; mean: number; - count: number; - sum: number; - std: number; - median: number; - majority: number; - minority: number; - unique: number; - histogram: [number[], number[]]; - valid_percent: number; - masked_pixels: number; - valid_pixels: number; - percentile_2: number; - percentile_98: number; + count?: number; + sum?: number; + std?: number; + median?: number; + majority?: number; + minority?: number; + unique?: number; + histogram?: [number[], number[]]; + valid_percent?: number; + masked_pixels?: number; + valid_pixels?: number; + percentile_2?: number; + percentile_98?: number; } export interface TimeseriesDataResult { @@ -247,97 +250,141 @@ async function requestTimeseries({ // attached yet. setTimeout(() => onData(layersBase), 0); + // TODO: Maybe there's a better way than an if else conditional? try { - const layerInfoFromSTAC = await queryClient.fetchQuery( - [TIMESERIES_DATA_BASE_ID, 'dataset', id, aoi, start, end], - ({ signal }) => - getDatasetAssets( - { - stacCol: layer.stacCol, - stacApiEndpoint: layer.stacApiEndpoint, - assets: layer.sourceParams?.assets || 'cog_default', - aoi, - dateStart: start, - dateEnd: end - }, - { signal }, - concurrencyManager - ), - { - staleTime: Infinity - } - ); - const { assets, ...otherCollectionProps } = layerInfoFromSTAC; - - if (assets.length > MAX_QUERY_NUM) - throw Error( - `Too many requests. We currently only allow requests up to ${MAX_QUERY_NUM} and this analysis requires ${assets.length} requests.` - ); - - onData({ - ...layersBase, - status: 'loading', - meta: { - total: assets.length, - loaded: 0 + if (layer.type === "arc") { + const params = { + collection_id: layer.stacCol, + variable: layer.sourceParams?.layers, + datetime_range: `${`${userTzDate2utcString(start).slice(0, -5)}Z`},${`${userTzDate2utcString(end).slice(0, -5)}Z`}`, + aoi: aoi } - }); - - const tileEndpointToUse = - layer.tileApiEndpoint ?? process.env.API_RASTER_ENDPOINT; - - const analysisParams = layersBase.layer.analysis?.sourceParams ?? {}; + const statistics = await queryClient.fetchQuery( + [TIMESERIES_DATA_BASE_ID, 'dataset', id], + async ({ signal }) => { + return concurrencyManager.queue(async () => { + const { data } = await axios.post( + `${layer.tileApiEndpoint}/statistics`, + params, + { signal } + ); + return data; + }); + }, + { + staleTime: Infinity + } + ); - const layerStatistics = await Promise.all( - assets.map(async ({ date, url }) => { - const statistics = await queryClient.fetchQuery( - [TIMESERIES_DATA_BASE_ID, 'asset', url], - async ({ signal }) => { - return concurrencyManager.queue(async () => { - const { data } = await axios.post( - `${tileEndpointToUse}/cog/statistics?url=${url}`, - // Making a request with a FC causes a 500 (as of 2023/01/20) - combineFeatureCollection(aoi), - { params: { ...analysisParams, url }, signal } - ); - return { - date, - // Remove 1 when https://github.com/NASA-IMPACT/veda-ui/issues/572 is fixed. - ...(data.properties.statistics.b1 || - data.properties.statistics['1']) - }; - }); - }, - { - staleTime: Infinity - } + console.log(statistics) + + onData({ + ...layersBase, + status: 'succeeded', + meta: { + total: statistics.length, + loaded: statistics.length + }, + data: { + // ...otherCollectionProps, + // TODO: Get these from the API instead + isPeriodic: false, + timeDensity: "year", + domain: ['1983-01-01T00:00:00Z', '2022-12-31T23:59:59Z'], + timeseries: statistics.slice(0, 4) // TODO: FIX: For some reason the UI freezes for more than 4 timestamps + } + }); + } else { + const layerInfoFromSTAC = await queryClient.fetchQuery( + [TIMESERIES_DATA_BASE_ID, 'dataset', id, aoi, start, end], + ({ signal }) => + getDatasetAssets( + { + stacCol: layer.stacCol, + stacApiEndpoint: layer.stacApiEndpoint, + assets: layer.sourceParams?.assets || 'cog_default', + aoi, + dateStart: start, + dateEnd: end + }, + { signal }, + concurrencyManager + ), + { + staleTime: Infinity + } + ); + const { assets, ...otherCollectionProps } = layerInfoFromSTAC; + + if (assets.length > MAX_QUERY_NUM) + throw Error( + `Too many requests. We currently only allow requests up to ${MAX_QUERY_NUM} and this analysis requires ${assets.length} requests.` ); - - onData({ - ...layersBase, - meta: { - total: assets.length, - loaded: (layersBase.meta.loaded ?? 0) + 1 - } - }); - - return statistics; - }) - ); - - onData({ - ...layersBase, - status: 'succeeded', - meta: { - total: assets.length, - loaded: assets.length - }, - data: { - ...otherCollectionProps, - timeseries: layerStatistics - } - }); + + onData({ + ...layersBase, + status: 'loading', + meta: { + total: assets.length, + loaded: 0 + } + }); + + const tileEndpointToUse = + layer.tileApiEndpoint ?? process.env.API_RASTER_ENDPOINT; + + const analysisParams = layersBase.layer.analysis?.sourceParams ?? {}; + + const layerStatistics = await Promise.all( + assets.map(async ({ date, url }) => { + const statistics = await queryClient.fetchQuery( + [TIMESERIES_DATA_BASE_ID, 'asset', url], + async ({ signal }) => { + return concurrencyManager.queue(async () => { + const { data } = await axios.post( + `${tileEndpointToUse}/cog/statistics?url=${url}`, + // Making a request with a FC causes a 500 (as of 2023/01/20) + combineFeatureCollection(aoi), + { params: { ...analysisParams, url }, signal } + ); + return { + date, + // Remove 1 when https://github.com/NASA-IMPACT/veda-ui/issues/572 is fixed. + ...(data.properties.statistics.b1 || + data.properties.statistics['1']) + }; + }); + }, + { + staleTime: Infinity + } + ); + + onData({ + ...layersBase, + meta: { + total: assets.length, + loaded: (layersBase.meta.loaded ?? 0) + 1 + } + }); + + return statistics; + }) + ); + onData({ + ...layersBase, + status: 'succeeded', + meta: { + total: assets.length, + loaded: assets.length + }, + data: { + ...otherCollectionProps, + timeseries: layerStatistics + } + }); + } } catch (error) { // Discard abort related errors. if (error.revert) return; From caf81b756719c2b142c34d2396d46fc743908d98 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Wed, 13 Mar 2024 17:59:33 -0500 Subject: [PATCH 3/4] Add example argis imageserver dataset --- mock/datasets/power_meteorology.data.mdx | 106 +++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 mock/datasets/power_meteorology.data.mdx diff --git a/mock/datasets/power_meteorology.data.mdx b/mock/datasets/power_meteorology.data.mdx new file mode 100644 index 000000000..93c53c048 --- /dev/null +++ b/mock/datasets/power_meteorology.data.mdx @@ -0,0 +1,106 @@ +--- +id: power_901_annual_meterology_utc +name: 'POWER meteorology' +featured: true +description: "Meteorology" +media: + src: ::file ./no2--dataset-cover.jpg + alt: Power plant shooting steam at the sky. + author: + name: Mick Truyts + url: https://unsplash.com/photos/x6WQeNYJC1w +taxonomy: + - name: Topics + values: + - Air Quality +layers: + - id: cdd10 + stacApiEndpoint: http://localhost:8000 + tileApiEndpoint: http://localhost:8000 + stacCol: POWER/power_901_annual_meterology_utc + name: Cooling Degree Days Above 10 C + type: arc + description: + "Lorem Ipsum" + zoomExtent: + - 0 + - 20 + sourceParams: + layers: CDD10 + legend: + unit: + label: degree-day + type: gradient + min: 0 + max: 8972.0625 + stops: + - "#4575b4" + - "#91bfdb" + - "#e0f3f8" + - "#ffffbf" + - "#fee090" + - "#fc8d59" + - "#d73027" + # analysis: + # exclude: true + - id: cdd18_3 + stacApiEndpoint: http://localhost:8000 + tileApiEndpoint: http://localhost:8000 + stacCol: POWER/power_901_annual_meterology_utc + name: Cooling Degree Days Above 18.3 C + type: arc + description: + "Lorem Ipsum" + zoomExtent: + - 0 + - 20 + sourceParams: + layers: CDD18_3 + # TODO: get this from `/legend` endpoint? ex - https://arcgis.asdc.larc.nasa.gov/server/rest/services/POWER/power_901_annual_meterology_utc/ImageServer/legend + legend: + unit: + label: degree-day + type: gradient + min: 0 + max: 8972.0625 + stops: + - "#4575b4" + - "#91bfdb" + - "#e0f3f8" + - "#ffffbf" + - "#fee090" + - "#fc8d59" + - "#d73027" + # analysis: + # exclude: true + - id: power_901_annual_meterology_utc_DISPH + stacApiEndpoint: http://localhost:8000 + tileApiEndpoint: http://localhost:8000 + stacCol: POWER/power_901_annual_meterology_utc + name: Zero Plane Displacement Height + type: arc + description: + "Lorem Ipsum" + zoomExtent: + - 0 + - 20 + sourceParams: + layers: DISPH + legend: + unit: + label: m + type: gradient + min: 0 + max: 8972.0625 + stops: + - "#4575b4" + - "#91bfdb" + - "#e0f3f8" + - "#ffffbf" + - "#fee090" + - "#fc8d59" + - "#d73027" + # analysis: + # exclude: true +--- + From 795704449b396d6017e7ea3ac1b0a9dbfe2ae131 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 14 Mar 2024 11:49:45 -0500 Subject: [PATCH 4/4] Update stac/tile urls --- mock/datasets/power_meteorology.data.mdx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mock/datasets/power_meteorology.data.mdx b/mock/datasets/power_meteorology.data.mdx index 93c53c048..376fbf1e9 100644 --- a/mock/datasets/power_meteorology.data.mdx +++ b/mock/datasets/power_meteorology.data.mdx @@ -15,8 +15,8 @@ taxonomy: - Air Quality layers: - id: cdd10 - stacApiEndpoint: http://localhost:8000 - tileApiEndpoint: http://localhost:8000 + stacApiEndpoint: https://hh76k5rqdh.execute-api.us-west-2.amazonaws.com/dev + tileApiEndpoint: https://hh76k5rqdh.execute-api.us-west-2.amazonaws.com/dev stacCol: POWER/power_901_annual_meterology_utc name: Cooling Degree Days Above 10 C type: arc @@ -44,8 +44,8 @@ layers: # analysis: # exclude: true - id: cdd18_3 - stacApiEndpoint: http://localhost:8000 - tileApiEndpoint: http://localhost:8000 + stacApiEndpoint: https://hh76k5rqdh.execute-api.us-west-2.amazonaws.com/dev + tileApiEndpoint: https://hh76k5rqdh.execute-api.us-west-2.amazonaws.com/dev stacCol: POWER/power_901_annual_meterology_utc name: Cooling Degree Days Above 18.3 C type: arc @@ -74,8 +74,8 @@ layers: # analysis: # exclude: true - id: power_901_annual_meterology_utc_DISPH - stacApiEndpoint: http://localhost:8000 - tileApiEndpoint: http://localhost:8000 + stacApiEndpoint: https://hh76k5rqdh.execute-api.us-west-2.amazonaws.com/dev + tileApiEndpoint: https://hh76k5rqdh.execute-api.us-west-2.amazonaws.com/dev stacCol: POWER/power_901_annual_meterology_utc name: Zero Plane Displacement Height type: arc