From 05336d280870d506763813ad7b643206b668c536 Mon Sep 17 00:00:00 2001 From: adrien guernier Date: Fri, 2 Feb 2024 11:02:31 +0100 Subject: [PATCH 01/14] Fix: exporting hidden resources without stream --- src/api/controller/api/hiddenResource.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/api/controller/api/hiddenResource.js b/src/api/controller/api/hiddenResource.js index 0bccaff89..d86229be4 100644 --- a/src/api/controller/api/hiddenResource.js +++ b/src/api/controller/api/hiddenResource.js @@ -1,19 +1,11 @@ import Koa from 'koa'; import route from 'koa-route'; -import fs from 'fs'; -import { unlinkFile } from '../../services/fsHelpers'; const app = new Koa(); const getExportHiddenResources = async ctx => { const hiddenResources = await ctx.hiddenResource.findAll(); - const date = new Date().getTime(); - const filePath = `/tmp/hiddenResources${date}.json`; - const stream = fs.createWriteStream(filePath); - stream.write(JSON.stringify(hiddenResources)); - stream.end(); - stream.on('finish', () => unlinkFile(filePath)); - ctx.body = fs.createReadStream(filePath); + ctx.body = JSON.stringify(hiddenResources); ctx.status = 200; }; From dc659c72c87d4a38c9426a90a0a5c273465fdd7a Mon Sep 17 00:00:00 2001 From: adrien guernier Date: Fri, 2 Feb 2024 15:57:09 +0100 Subject: [PATCH 02/14] Feat: wip importing hidden resource --- src/api/controller/api/hiddenResource.js | 41 ++++++++++++++ src/app/js/admin/api/hiddenResource.js | 13 ++++- .../js/admin/removedResources/ImportButton.js | 54 +++++++++++++++++++ .../removedResources/RemovedResourcePage.js | 8 ++- src/app/js/user/index.js | 11 ++++ 5 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 src/app/js/admin/removedResources/ImportButton.js diff --git a/src/api/controller/api/hiddenResource.js b/src/api/controller/api/hiddenResource.js index d86229be4..bfd87d13c 100644 --- a/src/api/controller/api/hiddenResource.js +++ b/src/api/controller/api/hiddenResource.js @@ -1,5 +1,10 @@ import Koa from 'koa'; import route from 'koa-route'; +import koaBodyParser from 'koa-bodyparser'; +import asyncBusboy from '@recuperateur/async-busboy'; +import mime from 'mime-types'; +import fs from 'fs'; +import { ObjectId } from 'mongodb'; const app = new Koa(); @@ -9,6 +14,42 @@ const getExportHiddenResources = async ctx => { ctx.status = 200; }; +const postImportHiddenResources = async ctx => { + const { files } = await asyncBusboy(ctx.req); + + if (files.length !== 1) { + ctx.status = 400; + ctx.body = { message: 'File does not exist' }; + return; + } + + const type = mime.lookup(files[0].filename); + if (type !== 'application/json') { + ctx.status = 400; + ctx.body = { message: 'Wrong mime type, application/json required' }; + return; + } + try { + const file = fs.readFileSync(files[0].path, 'utf8'); + const hiddenResources = JSON.parse(file); + await ctx.hiddenResource.deleteMany({}); + const toInsert = hiddenResources.map(hiddenResource => ({ + ...hiddenResource, + _id: new ObjectId(hiddenResource.id), + })); + await ctx.hiddenResource.insertMany(toInsert); + } catch (error) { + ctx.status = 400; + ctx.body = { message: error.message }; + return; + } + + ctx.body = { message: 'Imported' }; + ctx.status = 200; +}; + +app.use(koaBodyParser()); app.use(route.get('/export', getExportHiddenResources)); +app.use(route.post('/import', postImportHiddenResources)); export default app; diff --git a/src/app/js/admin/api/hiddenResource.js b/src/app/js/admin/api/hiddenResource.js index 5949bc4e8..02e6d0763 100644 --- a/src/app/js/admin/api/hiddenResource.js +++ b/src/app/js/admin/api/hiddenResource.js @@ -1,5 +1,5 @@ import { getUserSessionStorageInfo } from '../api/tools'; -import { getExportHiddenResources } from '../../user'; +import { getExportHiddenResources, getImportHiddenResources } from '../../user'; import fetch from '../../lib/fetch'; export const exportHiddenResources = () => { @@ -12,3 +12,14 @@ export const exportHiddenResources = () => { return response; }); }; + +export const importHiddenResources = async formData => { + const { token } = getUserSessionStorageInfo(); + const request = getImportHiddenResources({ token }, formData); + return fetch(request).then(({ response, error }) => { + if (error) { + return error; + } + return response; + }); +}; diff --git a/src/app/js/admin/removedResources/ImportButton.js b/src/app/js/admin/removedResources/ImportButton.js new file mode 100644 index 000000000..6e2cdb748 --- /dev/null +++ b/src/app/js/admin/removedResources/ImportButton.js @@ -0,0 +1,54 @@ +import React from 'react'; +import { compose } from 'recompose'; +import { polyglot as polyglotPropTypes } from '../../propTypes'; +import translate from 'redux-polyglot/translate'; +import UploadIcon from '@mui/icons-material/Upload'; +import { Button, styled } from '@mui/material'; +import { importHiddenResources } from '../api/hiddenResource'; + +const VisuallyHiddenInput = styled('input')({ + clip: 'rect(0 0 0 0)', + clipPath: 'inset(50%)', + height: 1, + overflow: 'hidden', + position: 'absolute', + bottom: 0, + left: 0, + whiteSpace: 'nowrap', + width: 1, +}); + +const ImportButton = ({ p: polyglot }) => { + const buttonLabel = polyglot.t('import'); + const handleFileChange = async event => { + const file = event.target.files[0]; + const formData = new FormData(); + formData.append('file', file); + const res = await importHiddenResources(formData); + console.log(res); + }; + + return ( + <> + + + ); +}; + +ImportButton.propTypes = { + p: polyglotPropTypes.isRequired, +}; + +export default compose(translate)(ImportButton); diff --git a/src/app/js/admin/removedResources/RemovedResourcePage.js b/src/app/js/admin/removedResources/RemovedResourcePage.js index 8d365cc72..af468554f 100644 --- a/src/app/js/admin/removedResources/RemovedResourcePage.js +++ b/src/app/js/admin/removedResources/RemovedResourcePage.js @@ -7,13 +7,19 @@ import RemovedResourceList from './RemovedResourceList'; import withInitialData from '../withInitialData'; import redirectToDashboardIfNoField from '../../admin/redirectToDashboardIfNoField'; import ExportButton from './ExportButton'; +import ImportButton from './ImportButton'; export const RemovedResourcePageComponent = ({ p: polyglot }) => { return ( {polyglot.t('hidden_resources')}} - action={} + action={ + <> + + + + } /> diff --git a/src/app/js/user/index.js b/src/app/js/user/index.js index 83967c671..039988553 100644 --- a/src/app/js/user/index.js +++ b/src/app/js/user/index.js @@ -649,6 +649,16 @@ export const getExportHiddenResources = state => { }); }; +export const getImportHiddenResources = (state, formData) => { + const req = getRequest(state, { + url: '/api/hiddenResource/import', + method: 'POST', + }); + delete req.headers['Content-Type']; + req.body = formData; + return req; +}; + export const selectors = { isAdmin, getRole, @@ -726,4 +736,5 @@ export const selectors = { getExportPrecomputedDataRequest, getPreviewPrecomputedDataRequest, getExportHiddenResources, + getImportHiddenResources, }; From 02b84a13e2d20b129b1e77f4797d9b367addfa85 Mon Sep 17 00:00:00 2001 From: adrien guernier Date: Fri, 2 Feb 2024 16:51:31 +0100 Subject: [PATCH 03/14] insert whole collection in database --- src/api/controller/api/hiddenResource.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/api/controller/api/hiddenResource.js b/src/api/controller/api/hiddenResource.js index bfd87d13c..642de7ff5 100644 --- a/src/api/controller/api/hiddenResource.js +++ b/src/api/controller/api/hiddenResource.js @@ -33,11 +33,7 @@ const postImportHiddenResources = async ctx => { const file = fs.readFileSync(files[0].path, 'utf8'); const hiddenResources = JSON.parse(file); await ctx.hiddenResource.deleteMany({}); - const toInsert = hiddenResources.map(hiddenResource => ({ - ...hiddenResource, - _id: new ObjectId(hiddenResource.id), - })); - await ctx.hiddenResource.insertMany(toInsert); + await ctx.hiddenResource.insertMany(hiddenResources); } catch (error) { ctx.status = 400; ctx.body = { message: error.message }; From 332ed333fd2862ccc1b25a34ebd500ca7f738bf1 Mon Sep 17 00:00:00 2001 From: adrien guernier Date: Fri, 2 Feb 2024 16:58:52 +0100 Subject: [PATCH 04/14] hide hidden resources in publishedDataset while importing hidden resources file --- src/api/controller/api/hiddenResource.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/api/controller/api/hiddenResource.js b/src/api/controller/api/hiddenResource.js index 642de7ff5..b76e3a787 100644 --- a/src/api/controller/api/hiddenResource.js +++ b/src/api/controller/api/hiddenResource.js @@ -34,6 +34,13 @@ const postImportHiddenResources = async ctx => { const hiddenResources = JSON.parse(file); await ctx.hiddenResource.deleteMany({}); await ctx.hiddenResource.insertMany(hiddenResources); + for (const hiddenResource of hiddenResources) { + await ctx.publishedDataset.hide( + hiddenResource.uri, + hiddenResource.reason, + hiddenResource.date, + ); + } } catch (error) { ctx.status = 400; ctx.body = { message: error.message }; From 6885b9a4bc9735f60e042114de9a004daa1a23e0 Mon Sep 17 00:00:00 2001 From: adrien guernier Date: Fri, 2 Feb 2024 17:18:10 +0100 Subject: [PATCH 05/14] add a CircularProgress when uploading --- src/app/js/admin/removedResources/ImportButton.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/app/js/admin/removedResources/ImportButton.js b/src/app/js/admin/removedResources/ImportButton.js index 6e2cdb748..60aba144e 100644 --- a/src/app/js/admin/removedResources/ImportButton.js +++ b/src/app/js/admin/removedResources/ImportButton.js @@ -1,9 +1,9 @@ -import React from 'react'; +import React, { useState } from 'react'; import { compose } from 'recompose'; import { polyglot as polyglotPropTypes } from '../../propTypes'; import translate from 'redux-polyglot/translate'; import UploadIcon from '@mui/icons-material/Upload'; -import { Button, styled } from '@mui/material'; +import { Button, CircularProgress, styled } from '@mui/material'; import { importHiddenResources } from '../api/hiddenResource'; const VisuallyHiddenInput = styled('input')({ @@ -20,12 +20,14 @@ const VisuallyHiddenInput = styled('input')({ const ImportButton = ({ p: polyglot }) => { const buttonLabel = polyglot.t('import'); + const [uploading, setUploading] = useState(false); const handleFileChange = async event => { + setUploading(true); const file = event.target.files[0]; const formData = new FormData(); formData.append('file', file); - const res = await importHiddenResources(formData); - console.log(res); + await importHiddenResources(formData); + setUploading(false); }; return ( @@ -34,7 +36,10 @@ const ImportButton = ({ p: polyglot }) => { component="label" variant="text" className="export" - startIcon={} + startIcon={ + uploading ? : + } + disabled={uploading} > {buttonLabel} Date: Fri, 2 Feb 2024 17:20:33 +0100 Subject: [PATCH 06/14] add Import translation --- src/app/custom/translations.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/custom/translations.tsv b/src/app/custom/translations.tsv index 1a4ba5c10..2f67c1e79 100644 --- a/src/app/custom/translations.tsv +++ b/src/app/custom/translations.tsv @@ -1064,3 +1064,4 @@ "root_panel_link" "Instances management" "Gestion des instances" "admin_panel_link" "Instance administration" "Administration de l'instance" "subresource_path_validation_error" "A subresource with this path already exists" "Une sous-ressource avec ce chemin existe déjà" +"import" "Import" "Importer" \ No newline at end of file From 3e369ba98048d93485b33a5195ddc950576a1539 Mon Sep 17 00:00:00 2001 From: adrien guernier Date: Fri, 2 Feb 2024 17:30:05 +0100 Subject: [PATCH 07/14] add a toast when upload is over --- src/app/custom/translations.tsv | 4 +++- src/app/js/admin/removedResources/ImportButton.js | 13 ++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/app/custom/translations.tsv b/src/app/custom/translations.tsv index 2f67c1e79..ded4ff06b 100644 --- a/src/app/custom/translations.tsv +++ b/src/app/custom/translations.tsv @@ -1064,4 +1064,6 @@ "root_panel_link" "Instances management" "Gestion des instances" "admin_panel_link" "Instance administration" "Administration de l'instance" "subresource_path_validation_error" "A subresource with this path already exists" "Une sous-ressource avec ce chemin existe déjà" -"import" "Import" "Importer" \ No newline at end of file +"import" "Import" "Importer" +"import_successfull" "Import completed successfully" "Import réalisé avec succés" +"import_successfull" "Error during import" "Erreur lors de l'import" \ No newline at end of file diff --git a/src/app/js/admin/removedResources/ImportButton.js b/src/app/js/admin/removedResources/ImportButton.js index 60aba144e..fd6a6afc8 100644 --- a/src/app/js/admin/removedResources/ImportButton.js +++ b/src/app/js/admin/removedResources/ImportButton.js @@ -5,6 +5,7 @@ import translate from 'redux-polyglot/translate'; import UploadIcon from '@mui/icons-material/Upload'; import { Button, CircularProgress, styled } from '@mui/material'; import { importHiddenResources } from '../api/hiddenResource'; +import { toast } from '../../../../common/tools/toast'; const VisuallyHiddenInput = styled('input')({ clip: 'rect(0 0 0 0)', @@ -26,8 +27,18 @@ const ImportButton = ({ p: polyglot }) => { const file = event.target.files[0]; const formData = new FormData(); formData.append('file', file); - await importHiddenResources(formData); + const response = await importHiddenResources(formData); setUploading(false); + + if (response.error) { + toast(polyglot.t('import_error'), { + type: toast.TYPE.ERROR, + }); + } else { + toast(polyglot.t('import_successfull'), { + type: toast.TYPE.SUCCESS, + }); + } }; return ( From d186a59d70328983cd842fe467bc8a88be864ffe Mon Sep 17 00:00:00 2001 From: adrien guernier Date: Fri, 2 Feb 2024 17:57:00 +0100 Subject: [PATCH 08/14] reload page when upload is done --- src/app/js/admin/removedResources/ImportButton.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/app/js/admin/removedResources/ImportButton.js b/src/app/js/admin/removedResources/ImportButton.js index fd6a6afc8..d44cd026c 100644 --- a/src/app/js/admin/removedResources/ImportButton.js +++ b/src/app/js/admin/removedResources/ImportButton.js @@ -1,4 +1,4 @@ -import React, { useState } from 'react'; +import React, { useEffect, useState } from 'react'; import { compose } from 'recompose'; import { polyglot as polyglotPropTypes } from '../../propTypes'; import translate from 'redux-polyglot/translate'; @@ -6,6 +6,7 @@ import UploadIcon from '@mui/icons-material/Upload'; import { Button, CircularProgress, styled } from '@mui/material'; import { importHiddenResources } from '../api/hiddenResource'; import { toast } from '../../../../common/tools/toast'; +import { useLocation, Redirect } from 'react-router-dom'; const VisuallyHiddenInput = styled('input')({ clip: 'rect(0 0 0 0)', @@ -21,7 +22,10 @@ const VisuallyHiddenInput = styled('input')({ const ImportButton = ({ p: polyglot }) => { const buttonLabel = polyglot.t('import'); + const location = useLocation(); const [uploading, setUploading] = useState(false); + const [done, setDone] = useState(false); + const handleFileChange = async event => { setUploading(true); const file = event.target.files[0]; @@ -29,18 +33,22 @@ const ImportButton = ({ p: polyglot }) => { formData.append('file', file); const response = await importHiddenResources(formData); setUploading(false); - if (response.error) { toast(polyglot.t('import_error'), { type: toast.TYPE.ERROR, }); } else { + setDone(true); toast(polyglot.t('import_successfull'), { type: toast.TYPE.SUCCESS, }); } }; + if (done) { + return ; + } + return ( <> - + ); };