From 9f0f095ef1bcd1bc64905858403ed846a6363104 Mon Sep 17 00:00:00 2001 From: mheggelund Date: Tue, 7 Nov 2023 09:34:15 +0100 Subject: [PATCH] feat: Add file chunking, model converting. --- src/pages/Browse/Browse.tsx | 173 ++++++++++++++++++++++++++++-------- 1 file changed, 137 insertions(+), 36 deletions(-) diff --git a/src/pages/Browse/Browse.tsx b/src/pages/Browse/Browse.tsx index 430c42f2..14440280 100644 --- a/src/pages/Browse/Browse.tsx +++ b/src/pages/Browse/Browse.tsx @@ -1,12 +1,19 @@ /* eslint-disable max-lines-per-function */ -import { Button, Snackbar, Typography } from '@equinor/eds-core-react'; +import { + Button, + LinearProgress, + Snackbar, + Typography, +} from '@equinor/eds-core-react'; import { useMutation } from '@tanstack/react-query'; -import { useState } from 'react'; +import { useEffect, useState } from 'react'; import { AnalogueModelsService, ConvertAnalogueModelCommand, CreateAnalogueModelCommand, JobsService, + UploadFileType, + UploadsService, } from '../../api/generated'; import { Table } from '../../components/Table'; import MetadataProps, { @@ -14,40 +21,49 @@ import MetadataProps, { } from '../../features/AddModel/AddModelDialog/AddModelDialog'; import * as Styled from './Browse.styled'; +const chunkSize = 1024 * 1024 * 99; // its 3MB, increase the number measure in mb enum UploadProcess { STARTED = 'We are uploading your new model. Please keep this browser tab open.', SUCCESS = 'Model successfully uploaded. You may close this browser tab now.', FAILED = 'File upload failed.', } -type MutationContract = { - id: string; - file: Blob; -}; - export const Browse = () => { + const [progress, setProgress] = useState(0); + + const [counter, setCounter] = useState(1); + const [fileToBeUpload, setFileToBeUpload] = useState(); + const [beginingOfTheChunk, setBeginingOfTheChunk] = useState(0); + const [endOfTheChunk, setEndOfTheChunk] = useState(chunkSize); + const [fileSize, setFileSize] = useState(0); + const [chunkCount, setChunkCount] = useState(0); + const [modelId, setModelId] = useState(''); + const [uploadId, setUploadId] = useState(''); + const [isAddModelDialog, setAddModelDialog] = useState(false); + const [uploadStatus, setUploadStatus] = useState(); + const createModel = useMutation({ mutationFn: AnalogueModelsService.postApiAnalogueModels, }); - const uploadNCFile = useMutation({ - mutationFn: (mutationContract: MutationContract) => { - return AnalogueModelsService.postApiAnalogueModelsNetcdfModels( - mutationContract.id, - { file: mutationContract.file }, - ); - }, + const modelManifest = useMutation({ + mutationFn: UploadsService.postApiUploadsModelsManifest, + }); + + const chunkUpload = useMutation({ + mutationFn: UploadsService.postApiUploadsModelsChunks, + }); + + const uploadFinished = useMutation({ + mutationFn: UploadsService.postApiUploadsModelsComplete, }); const convertModelFile = useMutation({ - mutationFn: (modelId: ConvertAnalogueModelCommand) => { - return JobsService.postApiJobsComputeModelConversions(modelId); + mutationFn: (requestBody: ConvertAnalogueModelCommand) => { + return JobsService.postApiJobsComputeModelConversions(requestBody); }, }); - const [isAddModelDialog, setAddModelDialog] = useState(false); - const [uploadStatus, setUploadStatus] = useState(); - function clearStatus() { setUploadStatus(undefined); } @@ -67,31 +83,112 @@ export const Browse = () => { const modelUpload = await createModel.mutateAsync(ModelBody); if (createModel.error === null && modelUpload.success) { - toggleDialog(); - const FileUploadBody: MutationContract = { - id: modelUpload.data.analogueModelId ?? '', - file: file, - }; - const fileUpload = await uploadNCFile.mutateAsync(FileUploadBody); + const id = modelUpload.data.analogueModelId; + setModelId(id); - if (uploadNCFile.error === null && fileUpload.success) { - setUploadStatus(UploadProcess.SUCCESS); + if (file === undefined) return; - const id = modelUpload.data.analogueModelId; - const convert = await convertModelFile.mutateAsync({ - modelId: id, - }); + const fileType = UploadFileType.NET_CDF; - // eslint-disable-next-line no-console - console.log(convert); - } else if (uploadNCFile.error) { - setUploadStatus(UploadProcess.FAILED); + const data = { + ModelId: id, + FileSize: file.size, + FileName: file.name, + FileExtension: 'test', + FileType: fileType, + }; - // TODO: show validation message + const createManifest = await modelManifest.mutateAsync(data); + if (modelManifest.error === null && createManifest.success) { + const chunkSize = createManifest.data.fileSize; + const uploadId = createManifest.data.uploadId; + setUploadId(uploadId); + + const _file = file; + setFileSize(_file.size); + const _totalCount = + _file.size % chunkSize === 0 + ? _file.size / chunkSize + : Math.floor(_file.size / chunkSize) + 1; // Total count of chunks will have been upload to finish the file + setChunkCount(_totalCount); + + setFileToBeUpload(_file); + toggleDialog(); } } } + const fileUpload = (counter: number) => { + setCounter(counter + 1); + if (counter <= chunkCount) { + if (fileToBeUpload === undefined) return; + const chunk = fileToBeUpload.slice(beginingOfTheChunk, endOfTheChunk); + const Blob = { Blob: chunk }; + + uploadChunk(Blob); + } + }; + + const uploadChunk = async ({ Blob }: { Blob: Blob }) => { + if (modelId === '' && uploadId === '') return; + const chunkData = { + ModelId: modelId, + UploadId: uploadId, + Blob: Blob, + ChunkNumber: counter, + }; + try { + const uploadChunks = await chunkUpload.mutateAsync(chunkData); + + if (chunkUpload.error === null && uploadChunks.success) { + setBeginingOfTheChunk(endOfTheChunk); + setEndOfTheChunk(endOfTheChunk + chunkSize); + if (counter === chunkCount) { + const finishBody = { + ModelId: modelId, + UploadId: uploadId, + }; + const finishedUpload = await uploadFinished.mutateAsync(finishBody); + if (uploadFinished.error === null && finishedUpload.success) { + setProgress(100); + console.log('Start Converting'); + + const convert = await convertModelFile.mutateAsync({ + modelId: modelId, + }); + + // eslint-disable-next-line max-depth + if (convertModelFile.error === null && convert.success) { + console.log('CONVERT FINISHED'); + setUploadStatus(UploadProcess.SUCCESS); + } else { + console.log('CONVERT FAILED'); + setUploadStatus(UploadProcess.FAILED); + } + } + } else { + const percentage = (counter / chunkCount) * 100; + setProgress(percentage); + } + } else { + console.log('Error Occurred:', uploadChunks.message); + } + } catch (error) { + console.log('error', error); + } + }; + + useEffect(() => { + if (fileSize > 0) { + fileUpload(counter); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [fileToBeUpload, progress]); + + useEffect(() => { + console.log(progress); + }, [progress]); + return ( <> @@ -106,6 +203,10 @@ export const Browse = () => { confirm={uploadModel} cancel={toggleDialog} /> + + File Upload Progress + +