[#1238] Create a derived atom for dataset hydration from Next.js (#1266)

**Related Ticket:** #1238
**Related Next.js PR:**
developmentseed/next-veda-ui#17

### Description of Changes
- Removed dependency on static datasets (veda_faux_module_datasets) when
reconciling datasets from URL params in E&A
- Created `externalDatasetsAtom` to store datasets from host
applications (eg Next.js)
- Modified `datasetLayersAtom` to use external datasets instead of the
bundled static datasets from VEDA UI

### Notes & Questions About Changes
_{Add additonal notes and outstanding questions here related to changes
in this pull request}_

### Validation / Testing
VEDA UI
1. Open E&A and select a few datasets
2. Once they are loaded, refresh the page
3. Verify that the datasets still load without issues, no errors are
thrown

Next.js
1. Same as for the VEDA UI above

app/scripts/components/exploration/atoms/datasetLayers.ts

@@ -0,0 +1,27 @@
+import { atom } from 'jotai';
+import { DatasetLayer } from '$types/veda';
+
+/**
+ * This is the primary storage atom for external datasets (e.g. passed from Next.js).
+ */
+export const externalDatasetsAtom = atom<any[]>([]);
+
+/**
+ * Derived atom that transforms the provided datasets into layers.
+ * It is used by the timelineDatasetsAtom to rebuild state from URL parameters
+ * while it preserves the parent dataset metadata for each layer that comes
+ * from the MDX configuration.
+ */
+export const datasetLayersAtom = atom<DatasetLayer[]>((get) => {
+  const datasets = get(externalDatasetsAtom);
+
+  return datasets.flatMap((dataset) => {
+    return (dataset.layers || []).map((l: any) => ({
+      ...l,
+      parentDataset: {
+        id: dataset.id,
+        name: dataset.name
+      }
+    }));
+  });
+});

app/scripts/components/exploration/atoms/datasets.ts

@@ -1,6 +1,6 @@
-import { datasetLayers } from '../data-utils';
 import { reconcileDatasets } from '../data-utils-no-faux-module';
 import { TimelineDataset, TimelineDatasetForUrl } from '../types.d.ts';
+import { datasetLayersAtom } from './datasetLayers';
 import { atomWithUrlValueStability } from '$utils/params-location-atom/atom-with-url-value-stability';
 
 function urlDatasetsDehydrate(datasets: TimelineDataset[]) {
@@ -38,7 +38,7 @@ export const timelineDatasetsAtom = atomWithUrlValueStability<
   dehydrate: (datasets) => {
     return urlDatasetsDehydrate(datasets);
   },
-  reconcile: (urlDatasets, storageDatasets) => {
+  reconcile: (urlDatasets, storageDatasets, get) => {
     // Reconcile what needs to be reconciled.
     const reconciledDatasets = urlDatasets.map((enc) => {
       // We only want to do this on load. If the dataset was already
@@ -49,10 +49,14 @@ export const timelineDatasetsAtom = atomWithUrlValueStability<
       if (readyDataset) {
         return readyDataset;
       }
+      const currentDatasetLayers = get(datasetLayersAtom);
       // Reconcile the dataset with the internal data (from VEDA config files)
       // and then add the url stored settings.
-      // @TODO - replace datasetLayers
-      const [reconciled] = reconcileDatasets([enc.id], datasetLayers, []);
+      const [reconciled] = reconcileDatasets(
+        [enc.id],
+        currentDatasetLayers,
+        []
+      );
       if (enc.settings) {
         reconciled.settings = enc.settings;
       }

app/scripts/components/exploration/container.tsx

@@ -1,12 +1,13 @@
 import React, { useState } from 'react';
 import { TourProvider } from '@reactour/tour';
 import { DevTools } from 'jotai-devtools';
-import { useAtom } from 'jotai';
+import { useAtom, useSetAtom } from 'jotai';
 import { PopoverTourComponent, TourManager } from './tour-manager';
 
 import { DatasetSelectorModal } from './components/dataset-selector-modal';
 import { allExploreDatasets } from './data-utils';
 import useTimelineDatasetAtom from './hooks/use-timeline-dataset-atom';
+import { externalDatasetsAtom } from './atoms/datasetLayers';
 import ExplorationAndAnalysis from '.';
 import { urlAtom } from '$utils/params-location-atom/url';
 import { DATASETS_PATH, EXPLORATION_PATH } from '$utils/routes';
@@ -32,6 +33,8 @@ const tourProviderStyles = {
 };
 
 export default function ExplorationAndAnalysisContainer() {
+  const setExternalDatasets = useSetAtom(externalDatasetsAtom);
+  setExternalDatasets(allExploreDatasets);
   const [timelineDatasets, setTimelineDatasets] = useTimelineDatasetAtom();
   const [datasetModalRevealed, setDatasetModalRevealed] = useState(
     !timelineDatasets.length
@@ -40,8 +43,8 @@ export default function ExplorationAndAnalysisContainer() {
   // @NOTE: When Exploration page is preloaded (ex. Linked with react-router)
   // atomWithLocation gets initialized outside of Exploration page and returns the previous page's value
   // We check if url Atom actually returns the values for exploration page here.
-  const [currentUrl]= useAtom(urlAtom);
-  if(!currentUrl.pathname?.includes(EXPLORATION_PATH)) return null;
+  const [currentUrl] = useAtom(urlAtom);
+  if (!currentUrl.pathname?.includes(EXPLORATION_PATH)) return null;
 
   const openModal = () => setDatasetModalRevealed(true);
   const closeModal = () => setDatasetModalRevealed(false);

app/scripts/components/exploration/hooks/use-timeline-dataset-atom.tsx

@@ -2,10 +2,10 @@ import { useAtom } from 'jotai';
 import { timelineDatasetsAtom } from '../atoms/datasets';
 import { TimelineDataset } from '../types.d.ts';
 
-export default function useTimelineDatasetAtom (): [
+export default function useTimelineDatasetAtom(): [
   TimelineDataset[],
   (datasets: TimelineDataset[]) => void
-]  {
+] {
   const [datasets, setDatasets] = useAtom(timelineDatasetsAtom);
   return [datasets, setDatasets];
-}
+}

app/scripts/index.ts

@@ -29,6 +29,10 @@ import useTimelineDatasetAtom from '$components/exploration/hooks/use-timeline-d
 import { timelineDatasetsAtom } from '$components/exploration/atoms/datasets';
 import { DatasetSelectorModal } from '$components/exploration/components/dataset-selector-modal';
 import { EnvConfigProvider } from '$context/env-config';
+import {
+  datasetLayersAtom,
+  externalDatasetsAtom
+} from '$components/exploration/atoms/datasetLayers';
 
 // Adding .last property to array
 /* eslint-disable-next-line fp/no-mutating-methods */
@@ -77,5 +81,7 @@ export {
   InternalNavLink,
 
   // STATE
-  timelineDatasetsAtom
+  timelineDatasetsAtom,
+  externalDatasetsAtom,
+  datasetLayersAtom
 };

app/scripts/utils/params-location-atom/atom-with-url-value-stability.ts

@@ -57,10 +57,15 @@ interface StableAtomOptions<Value, ValueUrl> {
    * the information that goes to the url needs to be simplified.
    * @param urlValue The value stored in the URL parameter after hydration.
    * @param storageValue The value stored in the atom.
+   * @param get A getter function to access other atom values during reconciliation.
    * @returns The reconciled value. If the function is not provided the urlValue
    * is considered the reconciled value.
    */
-  reconcile?: (urlValue: ValueUrl, storageValue: Value) => Value;
+  reconcile?: (
+    urlValue: ValueUrl,
+    storageValue: Value,
+    get: (atom: any) => any
+  ) => Value;
   /**
    * An optional function to compare two atom values for equality.
    * @param prev The previous atom value.
@@ -93,7 +98,7 @@ export function atomWithUrlValueStability<T, TUrl = T>(
       const storageValue = get(storage);
 
       // Reconcile the hydrated value with the storage value.
-      const reconciled = reconcile(hydrated, storageValue);
+      const reconciled = reconcile(hydrated, storageValue, get);
 
       // If the reconciled value is equal to the storage value, return the
       // storage value to ensure equality.