support uploading data with json file (#323)

* support upload json data

Signed-off-by: ryjiang <[email protected]>

* fix bug

Signed-off-by: ryjiang <[email protected]>

* update text

Signed-off-by: ryjiang <[email protected]>

---------

Signed-off-by: ryjiang <[email protected]>

client/src/components/customDialog/DialogTemplate.tsx

@@ -1,4 +1,4 @@
-import { FC, useEffect, useRef, useState } from 'react';
+import { FC, useRef } from 'react';
 import { useTranslation } from 'react-i18next';
 import {
   DialogContent,

client/src/components/uploader/Types.ts

@@ -1,3 +1,5 @@
+import { FILE_MIME_TYPE } from '@/consts';
+
 export interface UploaderProps {
   label: string;
   accept: string;
@@ -10,7 +12,11 @@ export interface UploaderProps {
   overSizeWarning?: string;
   setFileName: (fileName: string) => void;
   // handle uploader uploaded
-  handleUploadedData: (data: string, uploader: HTMLFormElement) => void;
+  handleUploadedData: (
+    data: string,
+    uploader: HTMLFormElement,
+    type: FILE_MIME_TYPE
+  ) => void;
   // handle uploader onchange
   handleUploadFileChange?: (file: File, uploader: HTMLFormElement) => void;
   handleUploadError?: () => void;

client/src/components/uploader/Uploader.tsx

@@ -3,6 +3,7 @@ import { FC, useContext, useRef } from 'react';
 import { rootContext } from '@/context';
 import CustomButton from '../customButton/CustomButton';
 import { UploaderProps } from './Types';
+import { FILE_MIME_TYPE } from '@/consts';
 
 const getStyles = makeStyles((theme: Theme) => ({
   btn: {},
@@ -22,6 +23,7 @@ const Uploader: FC<UploaderProps> = ({
   setFileName,
 }) => {
   const inputRef = useRef(null);
+  const type = useRef<FILE_MIME_TYPE>(FILE_MIME_TYPE.CSV);
   const classes = getStyles();
 
   const { openSnackBar } = useContext(rootContext);
@@ -33,7 +35,7 @@ const Uploader: FC<UploaderProps> = ({
     reader.onload = async e => {
       const data = reader.result;
       if (data) {
-        handleUploadedData(data as string, inputRef.current!);
+        handleUploadedData(data as string, inputRef.current!, type.current);
       }
     };
     // handle upload error
@@ -46,6 +48,9 @@ const Uploader: FC<UploaderProps> = ({
     uploader!.onchange = (e: Event) => {
       const target = e.target as HTMLInputElement;
       const file: File = (target.files as FileList)[0];
+      if (file) {
+        type.current = file.type as FILE_MIME_TYPE; // This will log the MIME type of the file
+      }
       const isSizeOverLimit = file && maxSize && maxSize < file.size;
 
       if (!file) {

client/src/consts/Util.ts

@@ -47,3 +47,8 @@ export const LOGICAL_OPERATORS = [
     label: 'JSON_CONTAINS',
   },
 ];
+
+export enum FILE_MIME_TYPE {
+  CSV = 'text/csv',
+  JSON = 'application/json',
+}

client/src/i18n/cn/insert.ts

@@ -2,9 +2,9 @@ const insertTrans = {
   import: '导入数据',
   targetTip: '放置数据的位置',
   file: '文件',
-  uploaderLabel: '选择CSV文件',
+  uploaderLabel: '选择CSV或者JSON文件',
   fileNamePlaceHolder: '未选择文件',
-  sample: 'CSV样本',
+  sample: '样本',
   noteTitle: '注意',
   notes: [
     `确保数据中的列名与Schema中的字段标签名相同。`,

client/src/i18n/en/insert.ts

@@ -2,14 +2,15 @@ const insertTrans = {
   import: 'Import Data',
   targetTip: 'Where to put your data',
   file: 'File',
-  uploaderLabel: 'Choose CSV File',
-  fileNamePlaceHolder: 'No file selected',
+  uploaderLabel: 'Select a .csv or .json file',
+  fileNamePlaceHolder: 'No file has been selected',
   sample: 'CSV Sample',
   noteTitle: 'Note',
   notes: [
-    `Make sure column names in the data are same as the field label names in Schema.`,
-    `Data size should be less than 150MB and the number of rows should be less than 100000, for the data to be imported properly.`,
-    `The "Import Data" option will only append new records. You cannot update existing records using this option.`,
+    `CSV or JSON file is supported`,
+    `Ensure data column names match field label names in Schema.`,
+    `Data should be <150MB and <100,000 rows for proper import.`,
+    `"Import Data" only appends new records; it doesn't update existing ones.`,
   ],
   overSizeWarning: 'File data size should less than {{size}}MB',
   isContainFieldNames: 'First row contains field names?',

client/src/pages/dialogs/insert/Dialog.tsx

@@ -12,10 +12,11 @@ import { parse } from 'papaparse';
 import { useTranslation } from 'react-i18next';
 import DialogTemplate from '@/components/customDialog/DialogTemplate';
 import icons from '@/components/icons/Icons';
-import { rootContext } from '@/context';
 import { Option } from '@/components/customSelector/Types';
 import { PartitionHttp } from '@/http';
+import { rootContext } from '@/context';
 import { combineHeadsAndData } from '@/utils';
+import { FILE_MIME_TYPE } from '@/consts';
 import InsertImport from './Import';
 import InsertPreview from './Preview';
 import InsertStatus from './Status';
@@ -76,6 +77,9 @@ const InsertContainer: FC<InsertContentProps> = ({
 
   // uploaded csv data (type: string)
   const [csvData, setCsvData] = useState<any[]>([]);
+  const [jsonData, setJsonData] = useState<
+    Record<string, unknown> | undefined
+  >();
 
   // handle changed table heads
   const [tableHeads, setTableHeads] = useState<string[]>([]);
@@ -95,7 +99,7 @@ const InsertContainer: FC<InsertContentProps> = ({
        * 2. must upload a csv file
        */
       const selectValid = collectionValue !== '' && partitionValue !== '';
-      const uploadValid = csvData.length > 0;
+      const uploadValid = csvData.length > 0 || typeof jsonData !== 'undefined';
       const condition = selectValid && uploadValid;
       setNextDisabled(!condition);
     }
@@ -106,7 +110,14 @@ const InsertContainer: FC<InsertContentProps> = ({
       const headsValid = tableHeads.every(h => h !== '');
       setNextDisabled(!headsValid);
     }
-  }, [activeStep, collectionValue, partitionValue, csvData, tableHeads]);
+  }, [
+    activeStep,
+    collectionValue,
+    partitionValue,
+    csvData,
+    tableHeads,
+    jsonData,
+  ]);
 
   useEffect(() => {
     const heads = isContainFieldNames
@@ -280,8 +291,18 @@ const InsertContainer: FC<InsertContentProps> = ({
     return !isLengthEqual;
   };
 
-  const handleUploadedData = (csv: string, uploader: HTMLFormElement) => {
-    const { data } = parse(csv);
+  const handleUploadedData = (
+    content: string,
+    uploader: HTMLFormElement,
+    type: FILE_MIME_TYPE
+  ) => {
+    // if json, just parse json to object
+    if (type === FILE_MIME_TYPE.JSON) {
+      setJsonData(JSON.parse(content));
+      setCsvData([]);
+      return;
+    }
+    const { data } = parse(content);
     // if uploaded csv contains heads, firstRowItems is the list of all heads
     const [firstRowItems = []] = data as string[][];
 
@@ -294,14 +315,22 @@ const InsertContainer: FC<InsertContentProps> = ({
       return;
     }
     setCsvData(data);
+    setJsonData(undefined);
   };
 
   const handleInsertData = async () => {
     // start loading
     setInsertStatus(InsertStatusEnum.loading);
-    // combine table heads and data
-    const tableData = isContainFieldNames ? csvData.slice(1) : csvData;
-    const data = combineHeadsAndData(tableHeads, tableData);
+
+    // process data
+    const data =
+      typeof jsonData !== 'undefined'
+        ? jsonData
+        : combineHeadsAndData(
+            tableHeads,
+            isContainFieldNames ? csvData.slice(1) : csvData
+          );
+
     const { result, msg } = await handleInsert(
       collectionValue,
       partitionValue,
@@ -320,9 +349,13 @@ const InsertContainer: FC<InsertContentProps> = ({
   };
 
   const handleNext = () => {
+    const isJSON = typeof jsonData !== 'undefined';
     switch (activeStep) {
       case InsertStepperEnum.import:
-        setActiveStep(activeStep => activeStep + 1);
+        setActiveStep(activeStep => activeStep + (isJSON ? 2 : 1));
+        if (isJSON) {
+          handleInsertData();
+        }
         break;
       case InsertStepperEnum.preview:
         setActiveStep(activeStep => activeStep + 1);

client/src/pages/dialogs/insert/Import.tsx

@@ -158,7 +158,7 @@ const InsertImport: FC<InsertImportProps> = ({
           <Uploader
             btnClass="uploader"
             label={insertTrans('uploaderLabel')}
-            accept=".csv"
+            accept=".csv,.json"
             // selected collection will affect schema, which is required for uploaded data validation check
             // so upload file should be disabled until user select one collection
             disabled={!selectedCollection}

client/src/pages/dialogs/insert/Types.ts

@@ -2,6 +2,7 @@ import { CollectionData } from '../../collections/Types';
 import { PartitionView } from '../../partitions/Types';
 import { FieldData } from '../../schema/Types';
 import { Option } from '@/components/customSelector/Types';
+import { FILE_MIME_TYPE } from '@/consts';
 
 export interface InsertContentProps {
   // optional on partition page since its collection is fixed
@@ -53,7 +54,11 @@ export interface InsertImportProps {
   handleCollectionChange?: (collectionName: string) => void;
   handlePartitionChange: (partitionName: string) => void;
   // handle uploaded data
-  handleUploadedData: (data: string, uploader: HTMLFormElement) => void;
+  handleUploadedData: (
+    data: string,
+    uploader: HTMLFormElement,
+    type: FILE_MIME_TYPE
+  ) => void;
   handleUploadFileChange: (file: File, uploader: HTMLFormElement) => void;
   fileName: string;
   setFileName: (fileName: string) => void;