Client-side fetch queries (#1252)

packages/toolpad-app/jest-environment-jsdom.ts

@@ -1,11 +1,19 @@
 import { TextDecoder, TextEncoder } from 'util';
 import fetch, { Headers, Request, Response } from 'node-fetch';
 import JsdomEnvironment from 'jest-environment-jsdom';
+import { RuntimeConfig } from './src/config';
+import { RUNTIME_CONFIG_WINDOW_PROPERTY } from './src/constants';
+
+function setRuntimeConfig(win: Window, config: RuntimeConfig) {
+  win[RUNTIME_CONFIG_WINDOW_PROPERTY] = config;
+}
 
 export default class CustomJsdomEnvironment extends JsdomEnvironment {
   async setup() {
     await super.setup();
 
+    setRuntimeConfig(this.global, { externalUrl: 'http://localhost:3000', isDemo: false });
+
     if (!this.global.TextDecoder) {
       // @ts-expect-error The polyfill is not 100% spec-compliant
       this.global.TextDecoder = TextDecoder;

packages/toolpad-app/src/appDom/index.ts

@@ -1048,7 +1048,13 @@ function createRenderTreeNode(node: AppDomNode): RenderTreeNode | null {
   }
 
   if (isQuery(node) || isMutation(node)) {
-    node = setNamespacedProp(node, 'attributes', 'query', null);
+    const isBrowserSideRestQuery: boolean =
+      node.attributes.dataSource?.value === 'rest' &&
+      !!(node.attributes.query.value as any).browser;
+
+    if (node.attributes.query.value && !isBrowserSideRestQuery) {
+      node = setNamespacedProp(node, 'attributes', 'query', null);
+    }
   }
 
   return node as RenderTreeNode;

packages/toolpad-app/src/runtime/ToolpadApp.tsx

@@ -460,7 +460,7 @@ function QueryNode({ node }: QueryNodeProps) {
 
   const configBindings = _.pick(node.attributes, USE_DATA_QUERY_CONFIG_KEYS);
   const options = resolveBindables(bindings, `${node.id}.config`, configBindings);
-  const queryResult = useDataQuery(node.id, params, options);
+  const queryResult = useDataQuery(node, params, options);
 
   React.useEffect(() => {
     const { isLoading, error, data, rows, ...result } = queryResult;

packages/toolpad-app/src/runtime/evalJsBindings.ts

@@ -1,31 +1,16 @@
 import { set } from 'lodash-es';
+import evalExpression from '../utils/evalExpression';
 import { mapValues } from '../utils/collections';
 import { errorFrom } from '../utils/errors';
 
-let iframe: HTMLIFrameElement;
-function evaluateCode(code: string, globalScope: Record<string, unknown>) {
-  // TODO: investigate https://www.npmjs.com/package/ses
-  if (!iframe) {
-    iframe = document.createElement('iframe');
-    iframe.setAttribute('sandbox', 'allow-same-origin allow-scripts');
-    iframe.style.display = 'none';
-    document.documentElement.appendChild(iframe);
-  }
-
-  // eslint-disable-next-line no-underscore-dangle
-  (iframe.contentWindow as any).__SCOPE = globalScope;
-  (iframe.contentWindow as any).console = window.console;
-  return (iframe.contentWindow as any).eval(`with (window.__SCOPE) { ${code} }`);
-}
-
 const TOOLPAD_LOADING_MARKER = '__TOOLPAD_LOADING_MARKER__';
 
 export function evaluateExpression(
   code: string,
   globalScope: Record<string, unknown>,
 ): BindingEvaluationResult {
   try {
-    const value = evaluateCode(code, globalScope);
+    const value = evalExpression(code, globalScope);
     return { value };
   } catch (rawError) {
     const error = errorFrom(rawError);

packages/toolpad-app/src/runtime/useDataQuery.ts

@@ -1,10 +1,11 @@
 import { GridRowsProp } from '@mui/x-data-grid-pro';
 import * as React from 'react';
 import { useQuery, UseQueryOptions } from '@tanstack/react-query';
-import { NodeId } from '@mui/toolpad-core';
 import { useAppContext } from './AppContext';
 import { VersionOrPreview } from '../types';
 import { CanvasHooksContext } from './CanvasHooksContext';
+import dataSources from '../toolpadDataSources/runtime';
+import * as appDom from '../appDom';
 
 interface ExecDataSourceQueryParams {
   signal?: AbortSignal;
@@ -59,7 +60,7 @@ const EMPTY_ARRAY: any[] = [];
 const EMPTY_OBJECT: any = {};
 
 export function useDataQuery(
-  queryId: NodeId,
+  node: appDom.QueryNode,
   params: any,
   {
     enabled = true,
@@ -68,6 +69,11 @@ export function useDataQuery(
 ): UseFetch {
   const { appId, version } = useAppContext();
   const { savedNodes } = React.useContext(CanvasHooksContext);
+  const queryId = node.id;
+  const query = node.attributes.query?.value;
+  const dataSourceId = node.attributes.dataSource?.value;
+
+  const dataSource = dataSourceId ? dataSources[dataSourceId] : null;
 
   // These are only used by the editor to invalidate caches whenever the query changes during editing
   const nodeHash: number | undefined = savedNodes ? savedNodes[queryId] : undefined;
@@ -81,15 +87,20 @@ export function useDataQuery(
     refetch,
   } = useQuery(
     [appId, version, nodeHash, queryId, params],
-    ({ signal }) =>
-      queryId &&
-      execDataSourceQuery({
-        signal,
-        appId,
-        version,
-        queryId,
-        params,
-      }),
+    ({ signal }) => {
+      if (!queryId) {
+        return null;
+      }
+
+      const fetchFromServer = () =>
+        execDataSourceQuery({ signal, appId, version, queryId, params });
+
+      if (query && dataSource?.exec) {
+        return dataSource?.exec(query, params, fetchFromServer);
+      }
+
+      return fetchFromServer();
+    },
     {
       ...options,
       enabled: isNodeAvailableOnServer && !!queryId && enabled,

packages/toolpad-app/src/toolpadDataSources/demo.tsx

@@ -0,0 +1,3 @@
+import config from '../config';
+
+export const MOVIES_API_DEMO_URL = new URL('/static/movies.json', config.externalUrl).href;

packages/toolpad-app/src/toolpadDataSources/function/client.tsx

@@ -25,7 +25,8 @@ import QueryInputPanel from '../QueryInputPanel';
 import { useEvaluateLiveBindingEntries } from '../../toolpad/AppEditor/useEvaluateLiveBinding';
 import useShortcut from '../../utils/useShortcut';
 import { tryFormat } from '../../utils/prettier';
-import config from '../../config';
+import useFetchPrivate from '../useFetchPrivate';
+import { MOVIES_API_DEMO_URL } from '../demo';
 
 const EVENT_INTERFACE_IDENTIFIER = 'ToolpadFunctionEvent';
 
@@ -84,7 +85,7 @@ function ConnectionParamsInput({
 
 const DEFAULT_MODULE = `export default async function ({ parameters }: ToolpadFunctionEvent) {
   console.info("Executing function with parameters:", parameters);
-  const url = new URL("${new URL('/static/movies.json', config.externalUrl).href}");
+  const url = new URL("${MOVIES_API_DEMO_URL}");
   url.searchParams.set("timestamp", String(Date.now()));
 
   const response = await fetch(String(url));
@@ -113,17 +114,19 @@ function QueryEditor({
     [paramsEditorLiveValue],
   );
 
+  const fetchPrivate = useFetchPrivate<FunctionPrivateQuery, FunctionResult>();
+  const fetchServerPreview = React.useCallback(
+    (query: FunctionQuery, params: Record<string, string>) =>
+      fetchPrivate({ kind: 'debugExec', query, params }),
+    [fetchPrivate],
+  );
+
   const [previewLogs, setPreviewLogs] = React.useState<LogEntry[]>([]);
   const [previewHar, setPreviewHar] = React.useState(() => createHarLog());
-  const { preview, runPreview: handleRunPreview } = useQueryPreview<
-    FunctionPrivateQuery,
-    FunctionResult
-  >(
-    {
-      kind: 'debugExec',
-      query: input.query,
-      params: previewParams,
-    },
+  const { preview, runPreview: handleRunPreview } = useQueryPreview(
+    fetchServerPreview,
+    input.query,
+    previewParams,
     {
       onPreview(result) {
         setPreviewLogs((existing) => [...existing, ...result.logs]);

packages/toolpad-app/src/toolpadDataSources/googleSheets/client.tsx

@@ -31,6 +31,7 @@ import ErrorAlert from '../../toolpad/AppEditor/PageEditor/ErrorAlert';
 import QueryInputPanel from '../QueryInputPanel';
 import SplitPane from '../../components/SplitPane';
 import useQueryPreview from '../useQueryPreview';
+import useFetchPrivate from '../useFetchPrivate';
 
 const EMPTY_ROWS: any[] = [];
 
@@ -126,13 +127,17 @@ function QueryEditor({
     [],
   );
 
-  const { preview, runPreview: handleRunPreview } = useQueryPreview<
-    GoogleSheetsPrivateQuery,
-    GoogleSheetsResult
-  >({
-    type: 'DEBUG_EXEC',
-    query: input.query,
-  });
+  const fetchPrivate = useFetchPrivate<GoogleSheetsPrivateQuery, GoogleSheetsResult>();
+  const fetchServerPreview = React.useCallback(
+    (query: GoogleSheetsApiQuery) => fetchPrivate({ type: 'DEBUG_EXEC', query }),
+    [fetchPrivate],
+  );
+
+  const { preview, runPreview: handleRunPreview } = useQueryPreview(
+    fetchServerPreview,
+    input.query,
+    {},
+  );
 
   const rawRows: any[] = preview?.data || EMPTY_ROWS;
   const columns: GridColDef[] = React.useMemo(() => parseColumns(inferColumns(rawRows)), [rawRows]);

packages/toolpad-app/src/toolpadDataSources/movies/client.tsx

@@ -12,6 +12,7 @@ import { Maybe } from '../../utils/types';
 import useQueryPreview from '../useQueryPreview';
 import { MoviesQuery, MoviesConnectionParams } from './types';
 import { FetchResult } from '../rest/types';
+import useFetchPrivate from '../useFetchPrivate';
 
 function withDefaults(value: Maybe<MoviesConnectionParams>): MoviesConnectionParams {
   return {
@@ -52,9 +53,17 @@ export function QueryEditor({
     [setInput],
   );
 
-  const { preview, runPreview: handleRunPreview } = useQueryPreview<MoviesQuery, FetchResult>({
-    genre: input.query.genre,
-  });
+  const fetchPrivate = useFetchPrivate<MoviesQuery, FetchResult>();
+  const fetchServerPreview = React.useCallback(
+    (query: MoviesQuery) => fetchPrivate(query),
+    [fetchPrivate],
+  );
+
+  const { preview, runPreview: handleRunPreview } = useQueryPreview(
+    fetchServerPreview,
+    { genre: input.query.genre },
+    {},
+  );
 
   return (
     <SplitPane split="vertical" size="50%" allowResize>

packages/toolpad-app/src/toolpadDataSources/postgres/client.tsx

@@ -170,14 +170,18 @@ function QueryEditor({
     [paramsEditorLiveValue],
   );
 
-  const { preview, runPreview: handleRunPreview } = useQueryPreview<
-    PostgresPrivateQuery,
-    PostgresResult
-  >({
-    kind: 'debugExec',
-    query: input.query,
-    params: previewParams,
-  });
+  const fetchPrivate = useFetchPrivate<PostgresPrivateQuery, PostgresResult>();
+  const fetchServerPreview = React.useCallback(
+    (query: PostgresQuery, params: Record<string, string>) =>
+      fetchPrivate({ kind: 'debugExec', query, params }),
+    [fetchPrivate],
+  );
+
+  const { preview, runPreview: handleRunPreview } = useQueryPreview(
+    fetchServerPreview,
+    input.query,
+    previewParams,
+  );
 
   const rawRows: any[] = preview?.data || EMPTY_ROWS;
   const columns: GridColDef[] = React.useMemo(() => parseColumns(inferColumns(rawRows)), [rawRows]);