- {task?.status === 'processing' && (
- <>
-
Upload in progress
-
- There is a uploading task in progress created at{' '}
- {format(new Date(task.createdAt), 'MMM d, yyyy HH:mm z')}.
-
- >
- )}
-
{task?.status === 'completed' && task?.errors.length === 0 && (
<>
Upload completed
diff --git a/client/src/containers/admin/data-uploader/component.tsx b/client/src/containers/admin/data-uploader/component.tsx
deleted file mode 100644
index 23a2a8acf..000000000
--- a/client/src/containers/admin/data-uploader/component.tsx
+++ /dev/null
@@ -1,40 +0,0 @@
-import { useState } from 'react';
-import { DownloadIcon } from '@heroicons/react/solid';
-
-import DataUploadError from 'containers/admin/data-upload-error';
-import DataUploader from 'containers/uploader';
-import { Anchor } from 'components/button';
-
-import type { Task } from 'types';
-
-const AdminDataPage: React.FC<{ task: Task }> = ({ task }) => {
- const [isUploading, setIsUploading] = useState(false);
-
- return (
-
-
-
-
- 1. Download the Excel template and fill it with your data.
-
-
}
- >
- Download template
-
-
-
-
2. Upload the filled Excel file.
-
- {!isUploading && task?.status === 'failed' &&
}
-
-
-
- );
-};
-
-export default AdminDataPage;
diff --git a/client/src/containers/admin/data-uploader/index.ts b/client/src/containers/admin/data-uploader/index.ts
deleted file mode 100644
index b404d7fd4..000000000
--- a/client/src/containers/admin/data-uploader/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export { default } from './component';
diff --git a/client/src/containers/admin/data-uploader/index.tsx b/client/src/containers/admin/data-uploader/index.tsx
new file mode 100644
index 000000000..ac77d56e9
--- /dev/null
+++ b/client/src/containers/admin/data-uploader/index.tsx
@@ -0,0 +1,48 @@
+import { DownloadIcon } from '@heroicons/react/solid';
+
+import DataUploadError from '@/containers/admin/data-upload-error';
+import { Anchor } from '@/components/button';
+import UploadTracker from '@/containers/uploader/tracker';
+import { useLasTask } from '@/hooks/tasks';
+import DataUploader from '@/containers/uploader';
+
+const AdminDataUploader: React.FC = () => {
+ const { data: lastTask } = useLasTask();
+
+ return (
+
+
+
+
+ 1. Download the Excel template and fill it with your data.
+
+
}
+ >
+ Download template
+
+
+
+
2. Upload the filled Excel file.
+ {lastTask?.status !== 'processing' && (
+
+
+
+ )}
+ {lastTask?.status === 'processing' && (
+
+
+
+ )}
+ {lastTask?.status === 'failed' &&
}
+
+
+
+ );
+};
+
+export default AdminDataUploader;
diff --git a/client/src/containers/uploader/index.ts b/client/src/containers/uploader/index.ts
deleted file mode 100644
index b404d7fd4..000000000
--- a/client/src/containers/uploader/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export { default } from './component';
diff --git a/client/src/containers/uploader/component.tsx b/client/src/containers/uploader/index.tsx
similarity index 65%
rename from client/src/containers/uploader/component.tsx
rename to client/src/containers/uploader/index.tsx
index 5a1ab080e..2949e886f 100644
--- a/client/src/containers/uploader/component.tsx
+++ b/client/src/containers/uploader/index.tsx
@@ -2,13 +2,14 @@ import { useCallback, useEffect, useState } from 'react';
import toast from 'react-hot-toast';
import classNames from 'classnames';
import { useRouter } from 'next/router';
+import { useQueryClient } from '@tanstack/react-query';
-import { useUploadDataSource } from 'hooks/sourcing-data';
-import { useLasTask } from 'hooks/tasks';
-import FileDropzone from 'components/file-dropzone';
+import { useUploadDataSource } from '@/hooks/sourcing-data';
+import { LAST_TASK_PARAMS, useLasTask } from '@/hooks/tasks';
import { env } from '@/env.mjs';
+import FileDropzone from '@/components/file-dropzone';
-import type { FileDropZoneProps } from 'components/file-dropzone/types';
+import type { FileDropZoneProps } from '@/components/file-dropzone/types';
import type { Task } from 'types';
type DataUploaderProps = {
@@ -29,7 +30,7 @@ const DataUploader: React.FC
= ({ variant = 'default', onUplo
const router = useRouter();
const uploadDataSource = useUploadDataSource();
const lastTask = useLasTask();
- const { refetch: refetchLastTask } = lastTask;
+ const queryClient = useQueryClient();
const handleOnDrop: FileDropZoneProps['onDrop'] = useCallback(
(acceptedFiles) => {
@@ -40,8 +41,16 @@ const DataUploader: React.FC = ({ variant = 'default', onUplo
});
uploadDataSource.mutate(formData, {
- onSuccess: () => {
- refetchLastTask();
+ onSuccess: ({ data }) => {
+ const { attributes, ...rest } = data;
+ queryClient.setQueryData(['tasks', LAST_TASK_PARAMS], {
+ data: [
+ {
+ ...rest,
+ ...attributes,
+ },
+ ],
+ });
},
onError: ({ response }) => {
const errors = response?.data?.errors;
@@ -53,7 +62,7 @@ const DataUploader: React.FC = ({ variant = 'default', onUplo
},
});
},
- [refetchLastTask, uploadDataSource],
+ [uploadDataSource, queryClient],
);
const handleFileRejected: FileDropZoneProps['onDropRejected'] = useCallback((rejectedFiles) => {
@@ -69,12 +78,9 @@ const DataUploader: React.FC = ({ variant = 'default', onUplo
// Status of the uploading process
const isUploading = uploadDataSource.isLoading;
const isWaiting = uploadDataSource.isSuccess && currentTaskId === lastTask.data?.id;
- const isProcessing = lastTask.data?.status === 'processing';
- const isWorking = isUploading || isWaiting || isProcessing;
- const isCompleted =
- !isWorking &&
- uploadDataSource.isSuccess &&
- (lastTask.data?.status === 'completed' || lastTask.data?.status === 'failed');
+ const isTaskFailed = lastTask.data?.status === 'failed';
+ const isWorking = (isUploading || isWaiting) && !isTaskFailed;
+ const isCompleted = !isWorking && uploadDataSource.isSuccess;
useEffect(() => {
if (!currentTaskId && lastTask.data?.id) {
@@ -83,20 +89,15 @@ const DataUploader: React.FC = ({ variant = 'default', onUplo
}, [currentTaskId, lastTask.data?.id]);
useEffect(() => {
- if (isCompleted && router.isReady) {
- router.reload();
- onUploadInProgress(false);
- }
+ if (isCompleted) onUploadInProgress?.(false);
}, [isCompleted, onUploadInProgress, router]);
useEffect(() => {
- if (isWorking && onUploadInProgress) {
- onUploadInProgress(true);
- }
+ if (isWorking && onUploadInProgress) onUploadInProgress?.(true);
}, [isWorking, onUploadInProgress]);
return (
-
+
= ({ variant = 'default', onUplo
isUploading={isWorking}
/>
-
- {isWorking && (
-
-
-
-
- {isUploading && 'Uploading file...'}
- {isWaiting && 'File uploaded successfully! Starting to process the data...'}
- {isProcessing && 'Processing file...'}
-
-
-
- )}
);
};
diff --git a/client/src/containers/uploader/tracker/index.tsx b/client/src/containers/uploader/tracker/index.tsx
new file mode 100644
index 000000000..dec4ac08e
--- /dev/null
+++ b/client/src/containers/uploader/tracker/index.tsx
@@ -0,0 +1,88 @@
+import { FC, useCallback, useState } from 'react';
+import { useQueryClient } from '@tanstack/react-query';
+
+import { cn } from '@/lib/utils';
+import { formatPercentage } from '@/utils/number-format';
+import useSocket from '@/hooks/socket';
+
+const STEPS_ORDER = [
+ 'VALIDATING_DATA',
+ 'GEOCODING',
+ 'IMPORTING_DATA',
+ 'CALCULATING_IMPACT',
+] as const;
+
+const STEPS_NAMES = {
+ VALIDATING_DATA: 'Validating Data',
+ IMPORTING_DATA: 'Importing Data',
+ GEOCODING: 'Geocoding',
+ CALCULATING_IMPACT: 'Calculating Impact',
+} as const;
+
+type ProgressTask = {
+ kind: 'DATA_IMPORT_PROGRESS';
+ data: Record<
+ keyof typeof STEPS_NAMES,
+ {
+ status: 'processing' | 'idle' | 'completed';
+ progress: number;
+ }
+ >;
+};
+
+export const UploadTracker: FC = () => {
+ const queryClient = useQueryClient();
+ const [tasksProgress, setTaskProgress] = useState
(undefined);
+
+ const onProgress = useCallback(({ data }: ProgressTask) => {
+ setTaskProgress(data);
+ }, []);
+
+ const onComplete = useCallback(() => {
+ queryClient.invalidateQueries(['tasks']);
+ queryClient.invalidateQueries(['sourcingLocations']);
+ }, [queryClient]);
+
+ const onFailure = useCallback(() => {
+ queryClient.invalidateQueries(['tasks']);
+ }, [queryClient]);
+
+ useSocket({
+ DATA_IMPORT_PROGRESS: onProgress,
+ DATA_IMPORT_COMPLETED: onComplete,
+ DATA_IMPORT_FAILURE: onFailure,
+ });
+
+ return (
+
+
+ {Object.keys(STEPS_NAMES)
+ .sort(
+ (a, b) =>
+ STEPS_ORDER.indexOf(a as keyof typeof STEPS_NAMES) -
+ STEPS_ORDER.indexOf(b as keyof typeof STEPS_NAMES),
+ )
+ .map((key: keyof typeof STEPS_NAMES) => (
+
+
+ {STEPS_NAMES[key]}
+
+ {`Progress: ${
+ tasksProgress?.[key]?.progress
+ ? formatPercentage(tasksProgress[key].progress / 100)
+ : '–'
+ }`}
+
+ ))}
+
+
+ );
+};
+
+export default UploadTracker;
diff --git a/client/src/hooks/socket/index.ts b/client/src/hooks/socket/index.ts
new file mode 100644
index 000000000..122eef668
--- /dev/null
+++ b/client/src/hooks/socket/index.ts
@@ -0,0 +1,57 @@
+import { useEffect, useRef } from 'react';
+import { io, SocketOptions, Socket } from 'socket.io-client';
+import { useSession } from 'next-auth/react';
+
+import { env } from '@/env.mjs';
+
+const useSocket = (
+ events: { [key: string]: (...args: unknown[]) => void } = {},
+ options?: SocketOptions,
+) => {
+ const socketRef = useRef(undefined);
+ const { data: { accessToken = undefined } = {} } = useSession();
+
+ useEffect(() => {
+ if (!accessToken) return () => {};
+
+ socketRef.current = io(env.NEXT_PUBLIC_API_URL, {
+ transports: ['websocket'],
+ auth: {
+ token: accessToken,
+ },
+ ...options,
+ });
+ }, [accessToken, options]);
+
+ useEffect(() => {
+ const socket = socketRef.current;
+
+ if (!socket) return () => {};
+
+ socket.connect();
+
+ return () => {
+ socket.disconnect();
+ };
+ }, []);
+
+ useEffect(() => {
+ const socket = socketRef.current;
+
+ if (!socket) return () => {};
+
+ Object.entries(events).forEach(([event, handler]) => {
+ socket.on(event, handler);
+ });
+
+ return () => {
+ Object.entries(events).forEach(([event, handler]) => {
+ socket.off(event, handler);
+ });
+ };
+ }, [events]);
+
+ return socketRef.current;
+};
+
+export default useSocket;
diff --git a/client/src/hooks/sourcing-data/index.ts b/client/src/hooks/sourcing-data/index.ts
index 29fd901aa..3bfb76e65 100644
--- a/client/src/hooks/sourcing-data/index.ts
+++ b/client/src/hooks/sourcing-data/index.ts
@@ -1,15 +1,18 @@
import { useMutation } from '@tanstack/react-query';
import { apiRawService } from 'services/api';
+import { Task } from '@/types';
-import type { UseMutationResult } from '@tanstack/react-query';
-
-type ApiResponse = { data: { id: string } };
-
-export function useUploadDataSource(): UseMutationResult {
- const importDataSource = (data) =>
+export function useUploadDataSource() {
+ const importDataSource = (data: FormData) =>
apiRawService
- .request({
+ .request<{
+ data: {
+ id: Task['id'];
+ type: Task['type'];
+ attributes: Omit;
+ };
+ }>({
method: 'POST',
data,
url: 'import/sourcing-data',
@@ -17,7 +20,7 @@ export function useUploadDataSource(): UseMutationResult {
})
.then((response) => response.data);
- return useMutation(importDataSource, {
+ return useMutation(importDataSource, {
mutationKey: ['importSourcingData'],
});
}
diff --git a/client/src/hooks/tasks/index.ts b/client/src/hooks/tasks/index.ts
index 30cb7cd32..ce9bed50f 100644
--- a/client/src/hooks/tasks/index.ts
+++ b/client/src/hooks/tasks/index.ts
@@ -21,6 +21,13 @@ const DEFAULT_QUERY_OPTIONS = {
refetchOnReconnect: false,
};
+export const LAST_TASK_PARAMS = {
+ 'page[size]': 1,
+ 'page[number]': 1,
+ sort: '-createdAt',
+ include: 'user',
+};
+
export const useTasks = (
params: Record = {},
options: UseQueryOptions = {},
@@ -45,18 +52,10 @@ export const useTasks = (
};
export const useLasTask = () => {
- const tasks = useTasks(
- {
- 'page[size]': 1,
- 'page[number]': 1,
- sort: '-createdAt',
- include: 'user',
- },
- {
- refetchInterval: 20000,
- refetchOnReconnect: true,
- },
- );
+ const tasks = useTasks(LAST_TASK_PARAMS, {
+ refetchInterval: 20000,
+ refetchOnReconnect: true,
+ });
return { ...tasks, data: tasks?.data?.data?.[0] } as UseQueryResult;
};
diff --git a/client/src/pages/data/index.tsx b/client/src/pages/data/index.tsx
index c6e69d30c..019aaa391 100644
--- a/client/src/pages/data/index.tsx
+++ b/client/src/pages/data/index.tsx
@@ -1,26 +1,27 @@
import { useMemo } from 'react';
import Head from 'next/head';
-import { useSourcingLocations } from 'hooks/sourcing-locations';
-import { useLasTask } from 'hooks/tasks';
-import AdminLayout from 'layouts/admin';
-import AdminDataUploader from 'containers/admin/data-uploader';
-import AdminDataTable from 'containers/admin/data-table';
-import Loading from 'components/loading';
-import Search from 'components/search';
+import { useSourcingLocations } from '@/hooks/sourcing-locations';
+import AdminLayout from '@/layouts/admin';
+import AdminDataUploader from '@/containers/admin/data-uploader';
+import AdminDataTable from '@/containers/admin/data-table';
+import Search from '@/components/search';
+import { useLasTask } from '@/hooks/tasks';
const AdminDataPage: React.FC = () => {
// Getting sourcing locations to check if there are any data
- const { data, isFetched, isLoading } = useSourcingLocations({
+ const { data, isFetched } = useSourcingLocations({
fields: 'updatedAt',
'page[number]': 1,
'page[size]': 1,
});
- // Getting last task to check if there is a processing task
- const { data: lastTask } = useLasTask();
+ const { data: lastTask, isFetched: lastTaskIsFetched } = useLasTask();
- const thereIsData = useMemo(() => data?.meta?.totalItems > 0, [data?.meta?.totalItems]);
+ const thereIsData = useMemo(
+ () => isFetched && data?.meta?.totalItems > 0,
+ [isFetched, data?.meta?.totalItems],
+ );
return (
{
Manage data | Landgriffon
- {(!isFetched || isLoading) && (
-
-
-
- )}
-
- {/* Content when empty, or upload is processing or failed */}
- {isFetched && !thereIsData && }
+ {thereIsData && lastTask?.status !== 'processing' && }
- {/* Content when data and upload is completed */}
- {isFetched && thereIsData && }
+ {(['processing', 'failed'].includes(lastTask?.status) ||
+ (!lastTask && lastTaskIsFetched)) && }
);
};
diff --git a/client/src/types.d.ts b/client/src/types.d.ts
index eb06176e4..7cb1b9386 100644
--- a/client/src/types.d.ts
+++ b/client/src/types.d.ts
@@ -361,13 +361,14 @@ export type MakePropOptional = Omit & Partial[];
user?: User;
data?: Record;
createdAt: string;
dismissedBy: string;
+ logs?: Record[];
};
// User
diff --git a/client/yarn.lock b/client/yarn.lock
index e9cdd9d02..0e1f06929 100644
--- a/client/yarn.lock
+++ b/client/yarn.lock
@@ -2556,6 +2556,13 @@ __metadata:
languageName: node
linkType: hard
+"@socket.io/component-emitter@npm:~3.1.0":
+ version: 3.1.1
+ resolution: "@socket.io/component-emitter@npm:3.1.1"
+ checksum: 93792eafb63ad15259ba00885c3cf4fdc01d969b1db10a273ccac70bed2373b5170cbc94682372d666a44e4ad8faeb176fb6cbaaeeb66c87231e2ff3d72583f9
+ languageName: node
+ linkType: hard
+
"@streamparser/json@npm:^0.0.12":
version: 0.0.12
resolution: "@streamparser/json@npm:0.0.12"
@@ -4953,7 +4960,7 @@ __metadata:
languageName: node
linkType: hard
-"debug@npm:4, debug@npm:^4.1.0, debug@npm:^4.1.1, debug@npm:^4.3.1, debug@npm:^4.3.2, debug@npm:^4.3.3, debug@npm:^4.3.4":
+"debug@npm:4, debug@npm:^4.1.0, debug@npm:^4.1.1, debug@npm:^4.3.1, debug@npm:^4.3.2, debug@npm:^4.3.3, debug@npm:^4.3.4, debug@npm:~4.3.1, debug@npm:~4.3.2":
version: 4.3.4
resolution: "debug@npm:4.3.4"
dependencies:
@@ -5292,6 +5299,26 @@ __metadata:
languageName: node
linkType: hard
+"engine.io-client@npm:~6.5.2":
+ version: 6.5.3
+ resolution: "engine.io-client@npm:6.5.3"
+ dependencies:
+ "@socket.io/component-emitter": ~3.1.0
+ debug: ~4.3.1
+ engine.io-parser: ~5.2.1
+ ws: ~8.11.0
+ xmlhttprequest-ssl: ~2.0.0
+ checksum: a72596fae99afbdb899926fccdb843f8fa790c69085b881dde121285a6935da2c2c665ebe88e0e6aa4285637782df84ac882084ff4892ad2430b059fc0045db0
+ languageName: node
+ linkType: hard
+
+"engine.io-parser@npm:~5.2.1":
+ version: 5.2.2
+ resolution: "engine.io-parser@npm:5.2.2"
+ checksum: 470231215f3136a9259efb1268bc9a71f789af4e8c74da8d3b49ceb149fe3cd5c315bf0cd13d2d8d9c8f0f051c6f93b68e8fa9c89a3b612b9217bf33765c943a
+ languageName: node
+ linkType: hard
+
"enhanced-resolve@npm:^5.10.0":
version: 5.10.0
resolution: "enhanced-resolve@npm:5.10.0"
@@ -7943,6 +7970,7 @@ __metadata:
recharts: 2.9.0
rooks: 7.14.1
sharp: 0.32.6
+ socket.io-client: 4.7.5
start-server-and-test: 1.14.0
tailwind-merge: 2.2.1
tailwindcss: 3.4.1
@@ -10785,6 +10813,28 @@ __metadata:
languageName: node
linkType: hard
+"socket.io-client@npm:4.7.5":
+ version: 4.7.5
+ resolution: "socket.io-client@npm:4.7.5"
+ dependencies:
+ "@socket.io/component-emitter": ~3.1.0
+ debug: ~4.3.2
+ engine.io-client: ~6.5.2
+ socket.io-parser: ~4.2.4
+ checksum: a6994b93a753d14292682ee97ba3c925c54b63e6fcb2ed5e0aa1d7c1d6164ed4a30d993f7eaaa3017ddf868ad0a1ab996badc8310129070136d84668789ee6c9
+ languageName: node
+ linkType: hard
+
+"socket.io-parser@npm:~4.2.4":
+ version: 4.2.4
+ resolution: "socket.io-parser@npm:4.2.4"
+ dependencies:
+ "@socket.io/component-emitter": ~3.1.0
+ debug: ~4.3.1
+ checksum: 61540ef99af33e6a562b9effe0fad769bcb7ec6a301aba5a64b3a8bccb611a0abdbe25f469933ab80072582006a78ca136bf0ad8adff9c77c9953581285e2263
+ languageName: node
+ linkType: hard
+
"socks-proxy-agent@npm:^7.0.0":
version: 7.0.0
resolution: "socks-proxy-agent@npm:7.0.0"
@@ -12238,6 +12288,28 @@ __metadata:
languageName: node
linkType: hard
+"ws@npm:~8.11.0":
+ version: 8.11.0
+ resolution: "ws@npm:8.11.0"
+ peerDependencies:
+ bufferutil: ^4.0.1
+ utf-8-validate: ^5.0.2
+ peerDependenciesMeta:
+ bufferutil:
+ optional: true
+ utf-8-validate:
+ optional: true
+ checksum: 316b33aba32f317cd217df66dbfc5b281a2f09ff36815de222bc859e3424d83766d9eb2bd4d667de658b6ab7be151f258318fb1da812416b30be13103e5b5c67
+ languageName: node
+ linkType: hard
+
+"xmlhttprequest-ssl@npm:~2.0.0":
+ version: 2.0.0
+ resolution: "xmlhttprequest-ssl@npm:2.0.0"
+ checksum: 1e98df67f004fec15754392a131343ea92e6ab5ac4d77e842378c5c4e4fd5b6a9134b169d96842cc19422d77b1606b8df84a5685562b3b698cb68441636f827e
+ languageName: node
+ linkType: hard
+
"y18n@npm:^4.0.0":
version: 4.0.3
resolution: "y18n@npm:4.0.3"