From c21d11692d7e5d78402a5efb87035e9dddc37e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morel=20Se=CC=81bastien?= Date: Wed, 14 Feb 2024 15:07:44 -0800 Subject: [PATCH] feat(import-app): stream import --- Makefile | 2 +- components/crystallize-import/package.json | 3 +- components/crystallize-import/pnpm-lock.yaml | 56 +++++++++++++++++++ .../src/contracts/form-submission.ts | 1 + .../src/contracts/ui-types.ts | 1 + .../src/core.server/import-runner.server.ts | 35 ++++++++++-- .../src/core.server/services.server.ts | 20 +++++++ .../crystallize-import/src/routes/_index.tsx | 1 + .../src/routes/api.import.stream.$id.ts | 22 ++++++++ .../src/routes/api.submit.ts | 16 +++++- .../crystallize-import/src/ui/import/App.tsx | 55 ++++++++++++++---- .../import/components/action-bar/Submit.tsx | 1 + package-lock.json | 35 ------------ pnpm-lock.yaml | 20 +++++++ 14 files changed, 213 insertions(+), 55 deletions(-) create mode 100644 components/crystallize-import/src/core.server/services.server.ts create mode 100644 components/crystallize-import/src/routes/api.import.stream.$id.ts delete mode 100644 package-lock.json create mode 100644 pnpm-lock.yaml diff --git a/Makefile b/Makefile index 43b915d..08eea7f 100644 --- a/Makefile +++ b/Makefile @@ -38,4 +38,4 @@ add-component-compliance-files: ## Add the compliance files into all the compone .PHONY: codeclean codeclean: ## Code Clean - @yarn prettier --write . + @pnpm run prettier --write . diff --git a/components/crystallize-import/package.json b/components/crystallize-import/package.json index 4d0d654..86cdbff 100644 --- a/components/crystallize-import/package.json +++ b/components/crystallize-import/package.json @@ -31,6 +31,7 @@ "react-icons": "^5.0.1", "react-tiny-popover": "^8.0.4", "read-excel-file": "^5.7.1", + "remix-utils": "^7.5.0", "uuid": "^9.0.1" }, "devDependencies": { @@ -38,10 +39,10 @@ "@types/d3": "^7.4.3", "@types/jsonwebtoken": "^9.0.5", "@types/react": "^18.2.55", - "autoprefixer": "^10.4.17", "@types/react-datepicker": "^4.19.6", "@types/react-dom": "^18.2.19", "@types/uuid": "^9.0.8", + "autoprefixer": "^10.4.17", "postcss-import": "^16.0.0", "prettier": "^3.2.5", "tailwindcss": "^3.4.1", diff --git a/components/crystallize-import/pnpm-lock.yaml b/components/crystallize-import/pnpm-lock.yaml index 5afeb1a..46ce88d 100644 --- a/components/crystallize-import/pnpm-lock.yaml +++ b/components/crystallize-import/pnpm-lock.yaml @@ -68,6 +68,9 @@ dependencies: read-excel-file: specifier: ^5.7.1 version: 5.7.1 + remix-utils: + specifier: ^7.5.0 + version: 7.5.0(@remix-run/node@2.6.0)(@remix-run/react@2.6.0)(react@18.2.0) uuid: specifier: ^9.0.1 version: 9.0.1 @@ -8016,6 +8019,51 @@ packages: unified: 10.1.2 dev: true + /remix-utils@7.5.0(@remix-run/node@2.6.0)(@remix-run/react@2.6.0)(react@18.2.0): + resolution: + { + integrity: sha512-vzjDhbdwK+3XxqL2LM7ZPzq0zyjb+77eAWbKhDqhnh0nlA11n3CmTtYzrZYpVf6h24+6m5mD6KnRfcdD3Zrl7w==, + } + engines: { node: '>=18.0.0' } + peerDependencies: + '@remix-run/cloudflare': ^2.0.0 + '@remix-run/deno': ^2.0.0 + '@remix-run/node': ^2.0.0 + '@remix-run/react': ^2.0.0 + '@remix-run/router': ^1.7.2 + crypto-js: ^4.1.1 + intl-parse-accept-language: ^1.0.0 + is-ip: ^5.0.1 + react: ^18.0.0 + zod: ^3.22.4 + peerDependenciesMeta: + '@remix-run/cloudflare': + optional: true + '@remix-run/deno': + optional: true + '@remix-run/node': + optional: true + '@remix-run/react': + optional: true + '@remix-run/router': + optional: true + crypto-js: + optional: true + intl-parse-accept-language: + optional: true + is-ip: + optional: true + react: + optional: true + zod: + optional: true + dependencies: + '@remix-run/node': 2.6.0(typescript@5.3.3) + '@remix-run/react': 2.6.0(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + react: 18.2.0 + type-fest: 4.10.2 + dev: false + /require-from-string@2.0.2: resolution: { @@ -8947,6 +8995,14 @@ packages: } dev: false + /type-fest@4.10.2: + resolution: + { + integrity: sha512-anpAG63wSpdEbLwOqH8L84urkL6PiVIov3EMmgIhhThevh9aiMQov+6Btx0wldNcvm4wV+e2/Rt1QdDwKHFbHw==, + } + engines: { node: '>=16' } + dev: false + /type-is@1.6.18: resolution: { diff --git a/components/crystallize-import/src/contracts/form-submission.ts b/components/crystallize-import/src/contracts/form-submission.ts index d1ab39b..3f6ecfe 100644 --- a/components/crystallize-import/src/contracts/form-submission.ts +++ b/components/crystallize-import/src/contracts/form-submission.ts @@ -1,4 +1,5 @@ export interface FormSubmission { + importId?: string; shapeIdentifier: string; folderPath: string; rows: Record[]; diff --git a/components/crystallize-import/src/contracts/ui-types.ts b/components/crystallize-import/src/contracts/ui-types.ts index deb60f6..c8d41c1 100644 --- a/components/crystallize-import/src/contracts/ui-types.ts +++ b/components/crystallize-import/src/contracts/ui-types.ts @@ -89,6 +89,7 @@ export type State = { loading?: boolean; done?: boolean; channels: Record; + importId: string; preflight?: { validCount: number; errorCount: number; diff --git a/components/crystallize-import/src/core.server/import-runner.server.ts b/components/crystallize-import/src/core.server/import-runner.server.ts index 66da60c..a3b7397 100644 --- a/components/crystallize-import/src/core.server/import-runner.server.ts +++ b/components/crystallize-import/src/core.server/import-runner.server.ts @@ -1,7 +1,13 @@ import { Bootstrapper, EVENT_NAMES, JsonSpec } from '@crystallize/import-utilities'; -import { error } from 'ajv/dist/vocabularies/applicator/dependencies'; +import { EventEmitter } from 'events'; +import util from 'util'; + +export const dump = (obj: any, depth?: number) => { + console.log(util.inspect(obj, false, depth, true)); +}; type Deps = { + emitter: EventEmitter; tenantIdentifier: string; sessionId: string | undefined; skipPublication?: boolean; @@ -24,10 +30,12 @@ type Subscriptons = { }; export const runImport = async ( + importUuid: string, spec: JsonSpec, { onItemCreated, onItemUpdated }: Subscriptons, - { tenantIdentifier, sessionId, skipPublication, verbose }: Deps, + { tenantIdentifier, sessionId, skipPublication, verbose, emitter }: Deps, ) => { + dump({ spec }, 200); return new Promise((resolve) => { const errors: any = []; const bootstrapper = new Bootstrapper(); @@ -50,13 +58,27 @@ export const runImport = async ( } if (onItemCreated) { - bootstrapper.on(EVENT_NAMES.ITEM_CREATED, (data) => onItemCreated(data).catch((err) => errors.push(err))); + bootstrapper.on(EVENT_NAMES.ITEM_CREATED, (data) => { + emitter.emit(importUuid, { + event: 'item-created', + data, + }); + onItemCreated(data).catch((err) => errors.push(err)); + }); } if (onItemUpdated) { - bootstrapper.on(EVENT_NAMES.ITEM_UPDATED, (data) => onItemUpdated(data).catch((err) => errors.push(err))); + bootstrapper.on(EVENT_NAMES.ITEM_UPDATED, (data) => { + emitter.emit(importUuid, data); + emitter.emit(importUuid, { + event: 'item-updated', + data, + }); + onItemUpdated(data).catch((err) => errors.push(err)); + }); } bootstrapper.on(EVENT_NAMES.DONE, () => { + emitter.emit(importUuid, 'done'); bootstrapper.kill(); if (errors.length > 0) { resolve({ @@ -69,8 +91,13 @@ export const runImport = async ( }); }); bootstrapper.on(EVENT_NAMES.ERROR, (err: any) => { + emitter.emit(importUuid, { + event: 'error', + data: err, + }); errors.push(err); }); + emitter.emit(importUuid, 'started'); bootstrapper.start(); }); }; diff --git a/components/crystallize-import/src/core.server/services.server.ts b/components/crystallize-import/src/core.server/services.server.ts new file mode 100644 index 0000000..efa983c --- /dev/null +++ b/components/crystallize-import/src/core.server/services.server.ts @@ -0,0 +1,20 @@ +import { EventEmitter } from 'events'; + +declare global { + // eslint-disable-next-line no-var + var __services: Awaited>; +} + +export const buildServices = async () => { + if (!global.__services) { + global.__services = await build(); + } + return global.__services; +}; + +const build = async () => { + const emitter = new EventEmitter(); + return { + emitter, + }; +}; diff --git a/components/crystallize-import/src/routes/_index.tsx b/components/crystallize-import/src/routes/_index.tsx index 2e62d0b..13aa00d 100644 --- a/components/crystallize-import/src/routes/_index.tsx +++ b/components/crystallize-import/src/routes/_index.tsx @@ -29,6 +29,7 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { export default function Index() { const { shapes, folders, flows, channels } = useLoaderData(); const initialState: State = { + importId: Math.random().toString(36).substring(7), shapes, folders, flows, diff --git a/components/crystallize-import/src/routes/api.import.stream.$id.ts b/components/crystallize-import/src/routes/api.import.stream.$id.ts new file mode 100644 index 0000000..6c2d6ef --- /dev/null +++ b/components/crystallize-import/src/routes/api.import.stream.$id.ts @@ -0,0 +1,22 @@ +import type { LoaderFunctionArgs } from '@remix-run/node'; +import { eventStream } from 'remix-utils/sse/server'; +import { buildServices } from '~/core.server/services.server'; + +export async function loader({ request, params }: LoaderFunctionArgs) { + const { emitter } = await buildServices(); + + return eventStream(request.signal, (send) => { + const handler = (data: any) => { + send({ event: 'log', data: JSON.stringify(data) }); + }; + emitter.addListener(params.id!, handler); + send({ event: 'init', data: 'Started' }); + const interval = setInterval(() => { + send({ event: 'ping', data: 'pong' }); + }, 10000); + return function clear() { + emitter.removeListener(params.id!, handler); + clearInterval(interval); + }; + }); +} diff --git a/components/crystallize-import/src/routes/api.submit.ts b/components/crystallize-import/src/routes/api.submit.ts index 0b2d5fb..7067b38 100644 --- a/components/crystallize-import/src/routes/api.submit.ts +++ b/components/crystallize-import/src/routes/api.submit.ts @@ -2,21 +2,30 @@ import { type ActionFunctionArgs, json } from '@remix-run/node'; import { FormSubmission } from '~/contracts/form-submission'; import { getCookieValue } from '~/core.server/auth.server'; import { runImport } from '~/core.server/import-runner.server'; +import { buildServices } from '~/core.server/services.server'; import { specFromFormSubmission } from '~/core.server/spec-from-form-submission.server'; import CrystallizeAPI from '~/core.server/use-cases/crystallize'; export const action = async ({ request }: ActionFunctionArgs) => { + const { emitter } = await buildServices(); const api = await CrystallizeAPI(request); const post: FormSubmission = await request.json(); const [validationRules, shapes] = await Promise.all([api.fetchValidationsSchema(), api.fetchShapes()]); try { + const importId = post.importId ?? Math.random().toString(36).substring(7); const spec = await specFromFormSubmission(post, shapes); const results = await runImport( + importId, spec, { onItemUpdated: async (item) => { - const push = (stageId: string) => - api.pushItemToFlow( + const push = async (stageId: string) => { + emitter.emit(importId, { + event: 'stage-push', + item, + stageId, + }); + await api.pushItemToFlow( { id: item.id, language: item.language, @@ -24,7 +33,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { }, stageId, ); - + }; const validate = validationRules?.[item.shape.identifier]?.validate; if (!validate) { // no validation @@ -48,6 +57,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { }, }, { + emitter, tenantIdentifier: api.apiClient.config.tenantIdentifier, skipPublication: !(post.doPublish === true), sessionId: getCookieValue(request, 'connect.sid'), diff --git a/components/crystallize-import/src/ui/import/App.tsx b/components/crystallize-import/src/ui/import/App.tsx index a40d265..3cd85ed 100644 --- a/components/crystallize-import/src/ui/import/App.tsx +++ b/components/crystallize-import/src/ui/import/App.tsx @@ -1,17 +1,30 @@ +import { useEventSource } from 'remix-utils/sse/react'; import { ActionBar } from './components/action-bar/ActionBar'; import { DataMatchingForm } from './components/DataForm'; import { FileChooser } from './components/FileChooser'; import { useImport } from './provider'; +import { useEffect, useState } from 'react'; export const App = () => { const { state, dispatch } = useImport(); const { shapes, folders } = state; + const [streamLogs, setStreamLogs] = useState([]); + const data = useEventSource(`/api/import/stream/${state.importId}`, { event: 'log' }); + + useEffect(() => { + if (state.done) { + setStreamLogs([]); + } + if (data) { + setStreamLogs((prev) => [...prev, data]); + } + }, [data, state.done]); + return (
- {!state.rows?.length ? (
{
)} - {state.loading && ( -
- {state.loading && ( - <> -
-
+
+
+
+
+
+ Bip bop, doing stuff... +
+
+
+
+
+

Stream logs

+
    + {streamLogs.map((log, i) => { + const decoded = JSON.parse(log); + return ( +
  • + 
    {JSON.stringify(decoded, undefined, 2)}
    +
    • + ); + })} +
+
- Bip bop, doing stuff... - - )} +
+
)} - {state.errors && state.errors.length > 0 && (

Errors: 

{JSON.stringify(state.errors, null, 2)}
)} + {state.done && ( +
+

Import completed

+
+ )} {state.preflight && (
diff --git a/components/crystallize-import/src/ui/import/components/action-bar/Submit.tsx b/components/crystallize-import/src/ui/import/components/action-bar/Submit.tsx index 82887e0..6674cdc 100644 --- a/components/crystallize-import/src/ui/import/components/action-bar/Submit.tsx +++ b/components/crystallize-import/src/ui/import/components/action-bar/Submit.tsx @@ -47,6 +47,7 @@ export const Submit = () => { dispatch.updateLoading(true); try { const post: FormSubmission = { + importId: state.importId, shapeIdentifier: state.selectedShape.identifier, folderPath: state.selectedFolder.tree?.path ?? '/', groupProductsBy: state.groupProductsBy, diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 5c3a539..0000000 --- a/package-lock.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "apps", - "lockfileVersion": 2, - "requires": true, - "packages": { - "": { - "devDependencies": { - "prettier": "3.2.5" - } - }, - "node_modules/prettier": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz", - "integrity": "sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==", - "dev": true, - "bin": { - "prettier": "bin/prettier.cjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" - } - } - }, - "dependencies": { - "prettier": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz", - "integrity": "sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==", - "dev": true - } - } -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..1e0a83d --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,20 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +devDependencies: + prettier: + specifier: 3.2.5 + version: 3.2.5 + +packages: + /prettier@3.2.5: + resolution: + { + integrity: sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==, + } + engines: { node: '>=14' } + hasBin: true + dev: true