support download sample csv data (#319)

* support download sample csv data Signed-off-by: ryjiang <[email protected]> * optimize csv file name Signed-off-by: ryjiang <[email protected]> * add size in the file name Signed-off-by: ryjiang <[email protected]> * change type Signed-off-by: ryjiang <[email protected]> --------- Signed-off-by: ryjiang <[email protected]>
zilliztech · Nov 21, 2023 · 97f6d84 · 97f6d84
1 parent 6839cd2
commit 97f6d84
Show file tree

Hide file tree

Showing 10 changed files with 142 additions and 37 deletions.
diff --git a/client/package.json b/client/package.json
@@ -7,7 +7,7 @@
   "private": true,
   "dependencies": {
     "@date-io/dayjs": "1.x",
-    "@json2csv/plainjs": "^7.0.1",
+    "@json2csv/plainjs": "^7.0.3",
     "@material-ui/core": "4.12.4",
     "@material-ui/icons": "^4.11.3",
     "@material-ui/lab": "4.0.0-alpha.61",
@@ -17,6 +17,7 @@
     "d3": "^7.8.5",
     "dayjs": "^1.11.9",
     "file-saver": "^2.0.5",
+    "filesaver": "^0.0.13",
     "i18next": "^20.3.1",
     "papaparse": "^5.4.1",
     "react": "^18.2.0",

diff --git a/client/src/http/Collection.ts b/client/src/http/Collection.ts
@@ -28,6 +28,7 @@ export class CollectionHttp extends BaseModel implements CollectionView {
   private id!: string;
   private loadedPercentage!: string;
   private createdTime!: string;
+  private csv!: string;
   private schema!: {
     fields: Field[];
     autoID: boolean;
@@ -251,4 +252,8 @@ export class CollectionHttp extends BaseModel implements CollectionView {
   get _schema() {
     return this.schema;
   }
+
+  get _csv() {
+    return this.csv;
+  }
 }
diff --git a/client/src/i18n/en/insert.ts b/client/src/i18n/en/insert.ts
@@ -31,7 +31,8 @@ const insertTrans = {
 
   importSampleData: 'Import sample data into {{collection}}',
   sampleDataSize: 'Choose sample data size',
-  importSampleDataDesc: `Import random data based on the collection's schema.`
+  importSampleDataDesc: `Import random data based on the collection's schema.`,
+  downloadSampleDataCSV: `Download Sample Data CSV`,
 };
 
 export default insertTrans;
diff --git a/client/src/pages/dialogs/ImportSampleDialog.tsx b/client/src/pages/dialogs/ImportSampleDialog.tsx
@@ -1,18 +1,25 @@
 import { makeStyles, Theme, Typography } from '@material-ui/core';
 import { FC, useState, useContext } from 'react';
 import { useTranslation } from 'react-i18next';
+import { saveAs } from 'file-saver';
 import DialogTemplate from '@/components/customDialog/DialogTemplate';
 import CustomSelector from '@/components/customSelector/CustomSelector';
+import CustomIconButton from '@/components/customButton/CustomIconButton';
 import { rootContext } from '@/context';
 import { InsertStatusEnum } from './insert/Types';
 import { CollectionHttp, MilvusHttp } from '@/http';
 import { LoadSampleParam } from './Types';
+import icons from '@/components/icons/Icons';
+const DownloadIcon = icons.download;
 
 const getStyles = makeStyles((theme: Theme) => {
   return {
     icon: {
       fontSize: '16px',
     },
+    downloadBtn: {
+      margin: theme.spacing(1.5, 1),
+    },
 
     selectors: {
       '& .selectorWrapper': {
@@ -23,6 +30,7 @@ const getStyles = makeStyles((theme: Theme) => {
         '& .selectLabel': {
           fontSize: '14px',
           lineHeight: '20px',
+
           color: theme.palette.attuDark.main,
         },
 
@@ -33,8 +41,13 @@ const getStyles = makeStyles((theme: Theme) => {
         },
       },
 
+      '& .actions': {
+        display: 'flex',
+        flexDirection: 'row',
+      },
+
       '& .selector': {
-        minWidth: '128px',
+        minWidth: theme.spacing(48),
       },
     },
   };
@@ -73,14 +86,24 @@ const ImportSampleDialog: FC<{ collection: string }> = props => {
 
   const handleImportSample = async (
     collectionName: string,
-    size: string
-  ): Promise<{ result: boolean; msg: string }> => {
+    size: string,
+    download: boolean = false
+  ): Promise<{ result: string | boolean; msg: string }> => {
     const param: LoadSampleParam = {
       collection_name: collectionName,
       size: size,
+      download,
     };
     try {
-      await CollectionHttp.importSample(collectionName, param);
+      const res = (await CollectionHttp.importSample(
+        collectionName,
+        param
+      )) as CollectionHttp;
+      if (download) {
+        const blob = new Blob([res._csv], { type: 'text/csv;charset=utf-8;' });
+        saveAs(blob, `${collectionName}.sample.${size}.csv`);
+        return { result: res._csv, msg: '' };
+      }
       await MilvusHttp.flush(collectionName);
       return { result: true, msg: '' };
     } catch (err: any) {
@@ -93,7 +116,11 @@ const ImportSampleDialog: FC<{ collection: string }> = props => {
     }
   };
 
-  const handleNext = async () => {
+  const onDownloadCSVClicked = async () => {
+    return await handleImportSample(props.collection, size, true);
+  };
+
+  const importData = async () => {
     if (insertStatus === InsertStatusEnum.success) {
       handleCloseDialog();
       return;
@@ -127,7 +154,7 @@ const ImportSampleDialog: FC<{ collection: string }> = props => {
           ? btnTrans('done')
           : insertStatus
       }
-      handleConfirm={handleNext}
+      handleConfirm={importData}
       confirmDisabled={insertStatus === InsertStatusEnum.loading}
       showActions={true}
       showCancel={false}
@@ -142,18 +169,27 @@ const ImportSampleDialog: FC<{ collection: string }> = props => {
             </Typography>
           </div>
 
-          <CustomSelector
-            label={insertTrans('sampleDataSize')}
-            options={sizeOptions}
-            wrapperClass="selector"
-            labelClass="selectLabel"
-            value={size}
-            variant="filled"
-            onChange={(e: { target: { value: unknown } }) => {
-              const size = e.target.value;
-              setSize(size as string);
-            }}
-          />
+          <div className="actions">
+            <CustomSelector
+              label={insertTrans('sampleDataSize')}
+              options={sizeOptions}
+              wrapperClass="selector"
+              labelClass="selectLabel"
+              value={size}
+              variant="filled"
+              onChange={(e: { target: { value: unknown } }) => {
+                const size = e.target.value;
+                setSize(size as string);
+              }}
+            />
+            <CustomIconButton
+              className={classes.downloadBtn}
+              tooltip={insertTrans('downloadSampleDataCSV')}
+              onClick={onDownloadCSVClicked}
+            >
+              <DownloadIcon />
+            </CustomIconButton>
+          </div>
         </div>
       </form>
     </DialogTemplate>

diff --git a/client/src/pages/dialogs/Types.ts b/client/src/pages/dialogs/Types.ts
@@ -37,4 +37,5 @@ export interface LoadSampleParam {
   collection_name: string;
   // e.g. [{vector: [1,2,3], age: 10}]
   size: string;
+  download?: boolean;
 }
diff --git a/client/yarn.lock b/client/yarn.lock
@@ -484,18 +484,18 @@
     "@jridgewell/resolve-uri" "3.1.0"
     "@jridgewell/sourcemap-codec" "1.4.14"
 
-"@json2csv/formatters@^7.0.1":
-  version "7.0.1"
-  resolved "https://registry.yarnpkg.com/@json2csv/formatters/-/formatters-7.0.1.tgz#c025f0795f9bbab480de77e2248ab593987296b9"
-  integrity sha512-eCmYKIIoFDXUB0Fotet2RmcoFTtNLXLmSV7j6aEQH/D2GiO749Uan3ts03PtAhXpE11QghxBjS0toXom8VQNBw==
+"@json2csv/formatters@^7.0.3":
+  version "7.0.3"
+  resolved "https://registry.yarnpkg.com/@json2csv/formatters/-/formatters-7.0.3.tgz#4d0584bd2664d6dfaa7bccc192d1d18aac7bee27"
+  integrity sha512-QLTpBNmNxGDAQNALkWwPdnyJ7IdXHQ0Motzog4fZOKK2ozpxckl6vwhWNBxbQs/25Zhp7bLS4J6ILu/hh0en6w==
 
-"@json2csv/plainjs@^7.0.1":
-  version "7.0.1"
-  resolved "https://registry.yarnpkg.com/@json2csv/plainjs/-/plainjs-7.0.1.tgz#361d849f04a2a5013c7880738f08b6bc193c24eb"
-  integrity sha512-UAdaZwahrUeYhMYYilJwDsRfE7wDRsmGMsszYH67j8FLD5gZitqG38RXpUgHEH0s6YjsY8iKYWeEQ19WILncFA==
+"@json2csv/plainjs@^7.0.3":
+  version "7.0.3"
+  resolved "https://registry.yarnpkg.com/@json2csv/plainjs/-/plainjs-7.0.3.tgz#2182da06a38c17caceca64701389d9b80a233588"
+  integrity sha512-AEpEdeSu8o64Sdf7Xoy7BM4MyfZToN7oxxleXJ6u7v3h0V0hKdNIw7i4kQ08wiIe7lcJBhS3JRyEPKX6k2AsjA==
   dependencies:
-    "@json2csv/formatters" "^7.0.1"
-    "@streamparser/json" "^0.0.15"
+    "@json2csv/formatters" "^7.0.3"
+    "@streamparser/json" "^0.0.17"
     lodash.get "^4.4.2"
 
 "@material-ui/[email protected]":
@@ -636,10 +636,10 @@
   resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553"
   integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg==
 
-"@streamparser/json@^0.0.15":
-  version "0.0.15"
-  resolved "https://registry.yarnpkg.com/@streamparser/json/-/json-0.0.15.tgz#405fbe94877ce0cbd3cf650b4d9186a0ec6acd0a"
-  integrity sha512-6oikjkMTYAHGqKmcC9leE4+kY4Ch4eiTImXUN/N4d2bNGBYs0LJ/tfxmpvF5eExSU7iiPlV9jYlADqvj3NWA3Q==
+"@streamparser/json@^0.0.17":
+  version "0.0.17"
+  resolved "https://registry.yarnpkg.com/@streamparser/json/-/json-0.0.17.tgz#b68742ebb49eec9c1fcc76cfa730dd74a5131382"
+  integrity sha512-mW54K6CTVJVLwXRB6kSS1xGWPmtTuXAStWnlvtesmcySgtop+eFPWOywBFPpJO4UD173epYsPSP6HSW8kuqN8w==
 
 "@svgr/babel-plugin-add-jsx-attribute@^5.4.0":
   version "5.4.0"
@@ -2209,6 +2209,14 @@ file-saver@^2.0.5:
   resolved "https://registry.yarnpkg.com/file-saver/-/file-saver-2.0.5.tgz#d61cfe2ce059f414d899e9dd6d4107ee25670c38"
   integrity sha512-P9bmyZ3h/PRG+Nzga+rbdI4OEpNDzAVyy74uVO9ATgzLK6VtAsYybF/+TOCvrc0MO793d6+42lLyZTw7/ArVzA==
 
+filesaver@^0.0.13:
+  version "0.0.13"
+  resolved "https://registry.yarnpkg.com/filesaver/-/filesaver-0.0.13.tgz#fa9b2ac1371d436fe5edc9285ed998d1e2782bee"
+  integrity sha512-ay2iShYJKmzKRPk89cgb14foqtCXcJIe5i+qdlSPAouKfBv7F2VZ0lxk9GjpcODe9p2YrXfi3Q+4CRn7ZDmleQ==
+  dependencies:
+    mkdirp "^0.5.0"
+    safename "0.0.4"
+
 fill-range@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
@@ -2963,6 +2971,18 @@ minimatch@^3.0.4, minimatch@^3.1.1:
   dependencies:
     brace-expansion "^1.1.7"
 
+minimist@^1.2.6:
+  version "1.2.8"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
+  integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
+
+mkdirp@^0.5.0:
+  version "0.5.6"
+  resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6"
+  integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==
+  dependencies:
+    minimist "^1.2.6"
+
 [email protected]:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
@@ -3416,6 +3436,11 @@ rw@1:
   resolved "https://registry.npmmirror.com/rw/-/rw-1.3.3.tgz#3f862dfa91ab766b14885ef4d01124bfda074fb4"
   integrity sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==
 
+[email protected]:
+  version "0.0.4"
+  resolved "https://registry.yarnpkg.com/safename/-/safename-0.0.4.tgz#b82c3b6db70d943a0582f9052fbfbfebbb589af5"
+  integrity sha512-+n4TsvESZKTXbHxOTSyQ0Q1JCXRb6MohgrqC2fbdALzTNQP/IhPOnCNRA4JPtagQq+6DD5ZsQ3lKMy57BYvwJA==
+
 "safer-buffer@>= 2.1.2 < 3.0.0":
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"

diff --git a/server/package.json b/server/package.json
@@ -12,6 +12,7 @@
     "url": "https://github.com/zilliztech/attu"
   },
   "dependencies": {
+    "@json2csv/plainjs": "^7.0.3",
     "@zilliz/milvus2-sdk-node": "2.3.4",
     "axios": "^1.4.0",
     "chalk": "^4.1.2",

diff --git a/server/src/collections/collections.service.ts b/server/src/collections/collections.service.ts
@@ -22,6 +22,7 @@ import {
   CompactReq,
   CountReq,
 } from '@zilliz/milvus2-sdk-node';
+import { Parser } from '@json2csv/plainjs';
 import { throwErrorFromSDK, findKeyValue, genRows, ROW_COUNT } from '../utils';
 import { QueryDto, ImportSampleDto, GetReplicasDto } from './dto';
 
@@ -307,15 +308,23 @@ export class CollectionsService {
   /**
    * Load sample data into collection
    */
-  async importSample({ collection_name, size }: ImportSampleDto) {
+  async importSample({ collection_name, size, download }: ImportSampleDto) {
     const collectionInfo = await this.describeCollection({ collection_name });
     const fields_data = genRows(
       collectionInfo.schema.fields,
       parseInt(size, 10),
       collectionInfo.schema.enable_dynamic_field
     );
 
-    return await this.insert({ collection_name, fields_data });
+    if (download) {
+      const parser = new Parser({});
+      const csv = parser.parse(fields_data);
+      // If download is true, return the generated data directly
+      return { csv };
+    } else {
+      // Otherwise, insert the data into the collection
+      return await this.insert({ collection_name, fields_data });
+    }
   }
 
   async getCompactionState(data: GetCompactionStateReq) {

diff --git a/server/src/collections/dto.ts b/server/src/collections/dto.ts
@@ -54,9 +54,11 @@ export class InsertDataDto {
 
 export class ImportSampleDto {
   @IsOptional()
-  readonly collection_name?: string;
+  readonly collection_name: string;
   @IsString()
   readonly size: string;
+  @IsBoolean()
+  readonly download?: boolean;
 }
 
 export class GetReplicasDto {

diff --git a/server/yarn.lock b/server/yarn.lock
@@ -738,6 +738,20 @@
     "@jridgewell/resolve-uri" "3.1.0"
     "@jridgewell/sourcemap-codec" "1.4.14"
 
+"@json2csv/formatters@^7.0.3":
+  version "7.0.3"
+  resolved "https://registry.yarnpkg.com/@json2csv/formatters/-/formatters-7.0.3.tgz#4d0584bd2664d6dfaa7bccc192d1d18aac7bee27"
+  integrity sha512-QLTpBNmNxGDAQNALkWwPdnyJ7IdXHQ0Motzog4fZOKK2ozpxckl6vwhWNBxbQs/25Zhp7bLS4J6ILu/hh0en6w==
+
+"@json2csv/plainjs@^7.0.3":
+  version "7.0.3"
+  resolved "https://registry.yarnpkg.com/@json2csv/plainjs/-/plainjs-7.0.3.tgz#2182da06a38c17caceca64701389d9b80a233588"
+  integrity sha512-AEpEdeSu8o64Sdf7Xoy7BM4MyfZToN7oxxleXJ6u7v3h0V0hKdNIw7i4kQ08wiIe7lcJBhS3JRyEPKX6k2AsjA==
+  dependencies:
+    "@json2csv/formatters" "^7.0.3"
+    "@streamparser/json" "^0.0.17"
+    lodash.get "^4.4.2"
+
 "@malept/cross-spawn-promise@^1.1.0":
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/@malept/cross-spawn-promise/-/cross-spawn-promise-1.1.1.tgz#504af200af6b98e198bce768bc1730c6936ae01d"
@@ -842,6 +856,11 @@
   resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553"
   integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg==
 
+"@streamparser/json@^0.0.17":
+  version "0.0.17"
+  resolved "https://registry.yarnpkg.com/@streamparser/json/-/json-0.0.17.tgz#b68742ebb49eec9c1fcc76cfa730dd74a5131382"
+  integrity sha512-mW54K6CTVJVLwXRB6kSS1xGWPmtTuXAStWnlvtesmcySgtop+eFPWOywBFPpJO4UD173epYsPSP6HSW8kuqN8w==
+
 "@szmarczak/http-timer@^4.0.5":
   version "4.0.6"
   resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-4.0.6.tgz#b4a914bb62e7c272d4e5989fe4440f812ab1d807"
@@ -3430,6 +3449,11 @@ lodash.camelcase@^4.3.0:
   resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6"
   integrity sha1-soqmKIorn8ZRA1x3EfZathkDMaY=
 
+lodash.get@^4.4.2:
+  version "4.4.2"
+  resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99"
+  integrity sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==
+
 [email protected]:
   version "4.1.2"
   resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe"