From 4e379acc5f6351c2d20ae6935a2a631b784f627f Mon Sep 17 00:00:00 2001 From: ryjiang Date: Fri, 24 Nov 2023 11:28:13 +0800 Subject: [PATCH] support uploading data with json file (#323) * support upload json data Signed-off-by: ryjiang * fix bug Signed-off-by: ryjiang * update text Signed-off-by: ryjiang --------- Signed-off-by: ryjiang --- .../customDialog/DialogTemplate.tsx | 2 +- client/src/components/uploader/Types.ts | 8 ++- client/src/components/uploader/Uploader.tsx | 7 ++- client/src/consts/Util.ts | 5 ++ client/src/i18n/cn/insert.ts | 4 +- client/src/i18n/en/insert.ts | 11 ++-- client/src/pages/dialogs/insert/Dialog.tsx | 51 +++++++++++++++---- client/src/pages/dialogs/insert/Import.tsx | 2 +- client/src/pages/dialogs/insert/Types.ts | 7 ++- 9 files changed, 76 insertions(+), 21 deletions(-) diff --git a/client/src/components/customDialog/DialogTemplate.tsx b/client/src/components/customDialog/DialogTemplate.tsx index 49b8aa87..031f382d 100644 --- a/client/src/components/customDialog/DialogTemplate.tsx +++ b/client/src/components/customDialog/DialogTemplate.tsx @@ -1,4 +1,4 @@ -import { FC, useEffect, useRef, useState } from 'react'; +import { FC, useRef } from 'react'; import { useTranslation } from 'react-i18next'; import { DialogContent, diff --git a/client/src/components/uploader/Types.ts b/client/src/components/uploader/Types.ts index 97a375b6..1a174a19 100644 --- a/client/src/components/uploader/Types.ts +++ b/client/src/components/uploader/Types.ts @@ -1,3 +1,5 @@ +import { FILE_MIME_TYPE } from '@/consts'; + export interface UploaderProps { label: string; accept: string; @@ -10,7 +12,11 @@ export interface UploaderProps { overSizeWarning?: string; setFileName: (fileName: string) => void; // handle uploader uploaded - handleUploadedData: (data: string, uploader: HTMLFormElement) => void; + handleUploadedData: ( + data: string, + uploader: HTMLFormElement, + type: FILE_MIME_TYPE + ) => void; // handle uploader onchange handleUploadFileChange?: (file: File, uploader: HTMLFormElement) => void; handleUploadError?: () => void; diff --git a/client/src/components/uploader/Uploader.tsx b/client/src/components/uploader/Uploader.tsx index 6872fb9f..dd98da4e 100644 --- a/client/src/components/uploader/Uploader.tsx +++ b/client/src/components/uploader/Uploader.tsx @@ -3,6 +3,7 @@ import { FC, useContext, useRef } from 'react'; import { rootContext } from '@/context'; import CustomButton from '../customButton/CustomButton'; import { UploaderProps } from './Types'; +import { FILE_MIME_TYPE } from '@/consts'; const getStyles = makeStyles((theme: Theme) => ({ btn: {}, @@ -22,6 +23,7 @@ const Uploader: FC = ({ setFileName, }) => { const inputRef = useRef(null); + const type = useRef(FILE_MIME_TYPE.CSV); const classes = getStyles(); const { openSnackBar } = useContext(rootContext); @@ -33,7 +35,7 @@ const Uploader: FC = ({ reader.onload = async e => { const data = reader.result; if (data) { - handleUploadedData(data as string, inputRef.current!); + handleUploadedData(data as string, inputRef.current!, type.current); } }; // handle upload error @@ -46,6 +48,9 @@ const Uploader: FC = ({ uploader!.onchange = (e: Event) => { const target = e.target as HTMLInputElement; const file: File = (target.files as FileList)[0]; + if (file) { + type.current = file.type as FILE_MIME_TYPE; // This will log the MIME type of the file + } const isSizeOverLimit = file && maxSize && maxSize < file.size; if (!file) { diff --git a/client/src/consts/Util.ts b/client/src/consts/Util.ts index 3216ef89..1e17564b 100644 --- a/client/src/consts/Util.ts +++ b/client/src/consts/Util.ts @@ -47,3 +47,8 @@ export const LOGICAL_OPERATORS = [ label: 'JSON_CONTAINS', }, ]; + +export enum FILE_MIME_TYPE { + CSV = 'text/csv', + JSON = 'application/json', +} diff --git a/client/src/i18n/cn/insert.ts b/client/src/i18n/cn/insert.ts index 5680f1ca..864d99cb 100644 --- a/client/src/i18n/cn/insert.ts +++ b/client/src/i18n/cn/insert.ts @@ -2,9 +2,9 @@ const insertTrans = { import: '导入数据', targetTip: '放置数据的位置', file: '文件', - uploaderLabel: '选择CSV文件', + uploaderLabel: '选择CSV或者JSON文件', fileNamePlaceHolder: '未选择文件', - sample: 'CSV样本', + sample: '样本', noteTitle: '注意', notes: [ `确保数据中的列名与Schema中的字段标签名相同。`, diff --git a/client/src/i18n/en/insert.ts b/client/src/i18n/en/insert.ts index 516baebd..f15e6499 100644 --- a/client/src/i18n/en/insert.ts +++ b/client/src/i18n/en/insert.ts @@ -2,14 +2,15 @@ const insertTrans = { import: 'Import Data', targetTip: 'Where to put your data', file: 'File', - uploaderLabel: 'Choose CSV File', - fileNamePlaceHolder: 'No file selected', + uploaderLabel: 'Select a .csv or .json file', + fileNamePlaceHolder: 'No file has been selected', sample: 'CSV Sample', noteTitle: 'Note', notes: [ - `Make sure column names in the data are same as the field label names in Schema.`, - `Data size should be less than 150MB and the number of rows should be less than 100000, for the data to be imported properly.`, - `The "Import Data" option will only append new records. You cannot update existing records using this option.`, + `CSV or JSON file is supported`, + `Ensure data column names match field label names in Schema.`, + `Data should be <150MB and <100,000 rows for proper import.`, + `"Import Data" only appends new records; it doesn't update existing ones.`, ], overSizeWarning: 'File data size should less than {{size}}MB', isContainFieldNames: 'First row contains field names?', diff --git a/client/src/pages/dialogs/insert/Dialog.tsx b/client/src/pages/dialogs/insert/Dialog.tsx index 818f7ea4..ff01aeba 100644 --- a/client/src/pages/dialogs/insert/Dialog.tsx +++ b/client/src/pages/dialogs/insert/Dialog.tsx @@ -12,10 +12,11 @@ import { parse } from 'papaparse'; import { useTranslation } from 'react-i18next'; import DialogTemplate from '@/components/customDialog/DialogTemplate'; import icons from '@/components/icons/Icons'; -import { rootContext } from '@/context'; import { Option } from '@/components/customSelector/Types'; import { PartitionHttp } from '@/http'; +import { rootContext } from '@/context'; import { combineHeadsAndData } from '@/utils'; +import { FILE_MIME_TYPE } from '@/consts'; import InsertImport from './Import'; import InsertPreview from './Preview'; import InsertStatus from './Status'; @@ -76,6 +77,9 @@ const InsertContainer: FC = ({ // uploaded csv data (type: string) const [csvData, setCsvData] = useState([]); + const [jsonData, setJsonData] = useState< + Record | undefined + >(); // handle changed table heads const [tableHeads, setTableHeads] = useState([]); @@ -95,7 +99,7 @@ const InsertContainer: FC = ({ * 2. must upload a csv file */ const selectValid = collectionValue !== '' && partitionValue !== ''; - const uploadValid = csvData.length > 0; + const uploadValid = csvData.length > 0 || typeof jsonData !== 'undefined'; const condition = selectValid && uploadValid; setNextDisabled(!condition); } @@ -106,7 +110,14 @@ const InsertContainer: FC = ({ const headsValid = tableHeads.every(h => h !== ''); setNextDisabled(!headsValid); } - }, [activeStep, collectionValue, partitionValue, csvData, tableHeads]); + }, [ + activeStep, + collectionValue, + partitionValue, + csvData, + tableHeads, + jsonData, + ]); useEffect(() => { const heads = isContainFieldNames @@ -280,8 +291,18 @@ const InsertContainer: FC = ({ return !isLengthEqual; }; - const handleUploadedData = (csv: string, uploader: HTMLFormElement) => { - const { data } = parse(csv); + const handleUploadedData = ( + content: string, + uploader: HTMLFormElement, + type: FILE_MIME_TYPE + ) => { + // if json, just parse json to object + if (type === FILE_MIME_TYPE.JSON) { + setJsonData(JSON.parse(content)); + setCsvData([]); + return; + } + const { data } = parse(content); // if uploaded csv contains heads, firstRowItems is the list of all heads const [firstRowItems = []] = data as string[][]; @@ -294,14 +315,22 @@ const InsertContainer: FC = ({ return; } setCsvData(data); + setJsonData(undefined); }; const handleInsertData = async () => { // start loading setInsertStatus(InsertStatusEnum.loading); - // combine table heads and data - const tableData = isContainFieldNames ? csvData.slice(1) : csvData; - const data = combineHeadsAndData(tableHeads, tableData); + + // process data + const data = + typeof jsonData !== 'undefined' + ? jsonData + : combineHeadsAndData( + tableHeads, + isContainFieldNames ? csvData.slice(1) : csvData + ); + const { result, msg } = await handleInsert( collectionValue, partitionValue, @@ -320,9 +349,13 @@ const InsertContainer: FC = ({ }; const handleNext = () => { + const isJSON = typeof jsonData !== 'undefined'; switch (activeStep) { case InsertStepperEnum.import: - setActiveStep(activeStep => activeStep + 1); + setActiveStep(activeStep => activeStep + (isJSON ? 2 : 1)); + if (isJSON) { + handleInsertData(); + } break; case InsertStepperEnum.preview: setActiveStep(activeStep => activeStep + 1); diff --git a/client/src/pages/dialogs/insert/Import.tsx b/client/src/pages/dialogs/insert/Import.tsx index 30fbc571..759f2726 100644 --- a/client/src/pages/dialogs/insert/Import.tsx +++ b/client/src/pages/dialogs/insert/Import.tsx @@ -158,7 +158,7 @@ const InsertImport: FC = ({ void; handlePartitionChange: (partitionName: string) => void; // handle uploaded data - handleUploadedData: (data: string, uploader: HTMLFormElement) => void; + handleUploadedData: ( + data: string, + uploader: HTMLFormElement, + type: FILE_MIME_TYPE + ) => void; handleUploadFileChange: (file: File, uploader: HTMLFormElement) => void; fileName: string; setFileName: (fileName: string) => void;