From 3c378ec5efef4a824760e53e499b24961ea00e46 Mon Sep 17 00:00:00 2001 From: Clint Banzhaf Date: Thu, 10 Aug 2023 08:17:41 +0200 Subject: [PATCH] Folder Upload (#305) --- src/QtBase/Backend/FilesCreateResponse.cpp | 20 +- src/QtBase/Backend/FilesCreateResponse.h | 2 + .../BackendFileSystemWidget.cpp | 475 +++++++++++++++--- .../BackendFileSystemWidget.h | 12 +- 4 files changed, 444 insertions(+), 65 deletions(-) diff --git a/src/QtBase/Backend/FilesCreateResponse.cpp b/src/QtBase/Backend/FilesCreateResponse.cpp index d6befed7a..16ef5723c 100644 --- a/src/QtBase/Backend/FilesCreateResponse.cpp +++ b/src/QtBase/Backend/FilesCreateResponse.cpp @@ -17,7 +17,21 @@ using namespace Core; // constructor FilesCreateResponse::FilesCreateResponse(QNetworkReply* reply) : Response(reply) { - // check for errors - if (mHasError == true) - return; + // data + Json::Item dataItem = mJson.Find("data"); + if (dataItem.IsNull() == true) + { + mHasError = true; + return; + } + + // get the data chunk id + Json::Item fileIdItem = dataItem.Find("fileId"); + if (fileIdItem.IsNull() == true) + { + mHasError = true; + return; + } + + mFileId = fileIdItem.GetString(); } diff --git a/src/QtBase/Backend/FilesCreateResponse.h b/src/QtBase/Backend/FilesCreateResponse.h index 5bc1a6e02..10cabb0ba 100644 --- a/src/QtBase/Backend/FilesCreateResponse.h +++ b/src/QtBase/Backend/FilesCreateResponse.h @@ -17,6 +17,8 @@ class QTBASE_API FilesCreateResponse : public Response public: // constructor FilesCreateResponse(QNetworkReply* reply); + + const char* mFileId; }; diff --git a/src/Studio/Plugins/BackendFileSystem/BackendFileSystemWidget.cpp b/src/Studio/Plugins/BackendFileSystem/BackendFileSystemWidget.cpp index 87ae4f2e5..059035e7c 100644 --- a/src/Studio/Plugins/BackendFileSystem/BackendFileSystemWidget.cpp +++ b/src/Studio/Plugins/BackendFileSystem/BackendFileSystemWidget.cpp @@ -41,6 +41,8 @@ #include #include #include +#include +#include using namespace Core; @@ -303,7 +305,7 @@ void BackendFileSystemWidget::UpdateItems(QTreeWidgetItem* item) // reload file hiarchy from -void BackendFileSystemWidget::Refresh() +void BackendFileSystemWidget::Refresh(const QString& localfolder, const QString& cloudfolder, const bool xprun) { if (GetAuthenticationCenter()->IsInterfaceAllowed() == false) return; @@ -315,7 +317,7 @@ void BackendFileSystemWidget::Refresh() // 2. process request and connect to the reply QNetworkReply* reply = GetBackendInterface()->GetNetworkAccessManager()->ProcessRequest( request, Request::UIMODE_SILENT ); - connect(reply, &QNetworkReply::finished, this, [reply, this]() + connect(reply, &QNetworkReply::finished, this, [reply, this, localfolder, cloudfolder, xprun]() { QNetworkReply* networkReply = qobject_cast( sender() ); @@ -330,6 +332,10 @@ void BackendFileSystemWidget::Refresh() ReInit(); UpdateInterface(); + + // continue folder upload after folder creation (if any is pending) + if (!localfolder.isEmpty() && !cloudfolder.isEmpty()) + UploadFolder(localfolder, cloudfolder, xprun); }); } @@ -818,6 +824,14 @@ void BackendFileSystemWidget::OnContextMenu(const QPoint& point) QAction* saveToDiskAction = menu.addAction("Download", this, SLOT(OnSaveToDisk())); saveToDiskAction->setIcon(GetQtBaseManager()->FindIcon("Images/Icons/SaveAs.png")); } + + if (singleSelectedItem->GetCreud().Create() == true) + { + // upload JSON to cloud + QAction* uploadAction = menu.addAction("Upload", this, SLOT(OnUploadFromDisk())); + uploadAction->setIcon(GetQtBaseManager()->FindIcon("Images/Icons/Cloud.png")); + } + } else { @@ -856,10 +870,13 @@ void BackendFileSystemWidget::OnContextMenu(const QPoint& point) QAction* saveToDiskAction = menu.addAction("Download", this, SLOT(OnSaveToDisk()) ); saveToDiskAction->setIcon( GetQtBaseManager()->FindIcon("Images/Icons/SaveAs.png") ); + if (singleSelectedItem->GetCreud().Update() == true) + { // load JSON from disk and save it to the cloud - QAction* loadFromDiskSaveToCloudAction = menu.addAction("Upload local file to cloud", this, SLOT(OnLoadFromDiskAndSaveToCloud()) ); + QAction* loadFromDiskSaveToCloudAction = menu.addAction("Upload", this, SLOT(OnUploadFromDisk()) ); loadFromDiskSaveToCloudAction->setIcon( GetQtBaseManager()->FindIcon("Images/Icons/Cloud.png") ); - + } + menu.addSeparator(); // revisions @@ -1171,7 +1188,7 @@ void BackendFileSystemWidget::OnSaveToDisk() // download file FilesGetRequest request(GetUser()->GetToken(), rootModel.GetUuid()); QNetworkReply* reply = GetBackendInterface()->GetNetworkAccessManager()->ProcessRequest(request); - connect(reply, &QNetworkReply::finished, this, [reply, this, filename]() + connect(reply, &QNetworkReply::finished, this, [reply, this, filename, rootModel]() { Json json; QNetworkReply* networkReply = qobject_cast(sender()); @@ -1186,6 +1203,13 @@ void BackendFileSystemWidget::OnSaveToDisk() QMessageBox::warning(this, "Error", "JSON Parse failed", QMessageBox::Ok); return; } + + // add or update uuid + Core::String jsonstr; + auto jsonitm = json.GetRootItem().Find("uuid"); + if (!jsonitm.IsNull()) jsonitm.SetString(rootModel.GetUuid()); + else json.GetRootItem().AddString("uuid", rootModel.GetUuid()); + if (!json.WriteToFile(filename.toLatin1().data(), true)) { QMessageBox::warning(this, "Error", "File Write failed", QMessageBox::Ok); @@ -1254,7 +1278,7 @@ void BackendFileSystemWidget::OnSaveToDisk() const Core::String filename = path + model.GetNameString() + model.GetExtension(); FilesGetRequest request(GetUser()->GetToken(), model.GetUuid()); QNetworkReply* reply = GetBackendInterface()->GetNetworkAccessManager()->ProcessRequest(request); - connect(reply, &QNetworkReply::finished, this, [reply, this, filename]() + connect(reply, &QNetworkReply::finished, this, [reply, this, filename, model]() { Json json; QNetworkReply* networkReply = qobject_cast(sender()); @@ -1269,6 +1293,13 @@ void BackendFileSystemWidget::OnSaveToDisk() QMessageBox::warning(this, "Error", "JSON Parse failed", QMessageBox::Ok); return; } + + // add or update uuid + Core::String jsonstr; + auto jsonitm = json.GetRootItem().Find("uuid"); + if (!jsonitm.IsNull()) jsonitm.SetString(model.GetUuid()); + else json.GetRootItem().AddString("uuid", model.GetUuid()); + if (!json.WriteToFile(filename.AsChar(), true)) { QMessageBox::warning(this, "Error", "File Write failed", QMessageBox::Ok); @@ -1283,70 +1314,394 @@ void BackendFileSystemWidget::OnSaveToDisk() // load a JSON graph from disk and save it back to the backend -void BackendFileSystemWidget::OnLoadFromDiskAndSaveToCloud() +void BackendFileSystemWidget::OnUploadFromDisk() { - // get the filename where to save it - QFileDialog::Options options; - QString selectedFilter; - QString qtFilename = QFileDialog::getOpenFileName( this, // parent - "Open", // caption - "", // directory - "JSON (*.json)", - &selectedFilter, - options ); - - if (qtFilename.isEmpty() == true) - return; + const QList selectedItems = + mTreeWidget->selectedItems(); - String filename = FromQtString(qtFilename); + // only single selection supported + if (selectedItems.count() != 1) + return; - SelectionItem selectedItem = GetSelectedItem(); + QTreeWidgetItem* rootItem = selectedItems[0]; + SelectionItem rootModel = CreateSelectionItem(rootItem); - // is experience - if (selectedItem.GetTypeString().IsEqualNoCase(ITEM_TYPE_EXPERIENCE) == true) - { - Json jsonParser; - if (jsonParser.ParseFile(filename.AsChar()) == false) - { - LogError( "BackendFileSystemWidget::OnLoadFromDiskAndSaveToCloud(): Cannot open file '%s'.", filename.AsChar()); - return; - } + // upload single file + if (!rootModel.IsFolder()) + { + // show file selection dialog + const QString defaultName = mLastSelectedFileDialogFolder + '/' + rootModel.GetName() + rootModel.GetExtension().AsChar(); + const QString ext((rootModel.GetTypeString() + " (*" + rootModel.GetExtension() + ")").AsChar()); + const QString filename = QFileDialog::getOpenFileName(this, + "Upload", defaultName, ext); - Experience* experience = new Experience(); - if (experience->Load(jsonParser, jsonParser.GetRootItem()) == false) - { - LogError( "BackendFileSystemWidget::OnLoadFromDiskAndSaveToCloud(): Cannot parse Json file '%s'.", filename.AsChar()); - return; - } - - GetBackendInterface()->GetFileSystem()->SaveExperience( GetUser()->GetToken(), selectedItem.GetUuid(), experience ); + if (filename.isEmpty()) + return; - delete experience; - } - // is classifier/statemachine - else - { - Graph* graph = NULL; - - if (selectedItem.GetTypeString().IsEqualNoCase(ITEM_TYPE_CLASSIFIER) == true) - graph = new Classifier(); - else if (selectedItem.GetTypeString().IsEqualNoCase(ITEM_TYPE_STATEMACHINE) == true) - graph = new StateMachine(); - else - return; + // remember selected path + mLastSelectedFileDialogFolder = QFileInfo(filename).absolutePath(); - if (GraphImporter::LoadFromFile(filename.AsChar(), graph) == false) - { - LogError( "BackendFileSystemWidget::OnLoadFromDiskAndSaveToCloud(): Cannot parse Json file '%s'.", filename.AsChar()); - return; - } - - GetBackendInterface()->GetFileSystem()->SaveGraph( GetUser()->GetToken(), selectedItem.GetUuid(), graph ); + // load file + QFile f(filename); + if (!f.open(QFile::ReadOnly | QFile::Text)) { + QMessageBox::warning(this, "Error", "Failed to open file", QMessageBox::Ok); + return; + } + QTextStream in(&f); + QByteArray arr(f.readAll()); - delete graph; - } + // verify json + Json json; + if (!json.Parse(arr.constData())) { + QMessageBox::warning(this, "Error", "Failed to parse JSON", QMessageBox::Ok); + return; + } + + // replace a possible uuid with one from backend + Core::String jsonstr; + auto jsonitm = json.GetRootItem().Find("uuid"); + if (!jsonitm.IsNull()) + jsonitm.SetString(rootModel.GetUuid()); + json.WriteToString(jsonstr, true); + + // upload + FilesUpdateRequest request(GetUser()->GetToken(), rootModel.GetUuid(), jsonstr.AsChar()); + QNetworkReply* reply = GetBackendInterface()->GetNetworkAccessManager()->ProcessRequest(request); + connect(reply, &QNetworkReply::finished, this, [reply, this]() + { + QNetworkReply* networkReply = qobject_cast(sender()); + FilesUpdateResponse response(networkReply); + if (response.HasError()) + { + QMessageBox::warning(this, "Error", "Upload failed", QMessageBox::Ok); + return; + } + this->Refresh(); + }); + } + + // upload folder + else + { + // show folder selection dialog + const QString defaultName = mLastSelectedFileDialogFolder + '/' + rootModel.GetName(); + const QString folder = QFileDialog::getExistingDirectory( + this, "Select Folder", defaultName); + + if (folder.isEmpty()) + return; + + // remember selected path + mLastSelectedFileDialogFolder = QFileInfo(folder).absolutePath(); + + // shared root path of all elements + const Core::String& rootPath = rootModel.GetPathString(); + const uint32 rootPathLength = rootPath.GetLength(); + + // prepare upload + mPendingUploads = 0; + mLocalUploadRoot = folder; + mCloudUploadRoot = rootPath.AsChar(); + mLookup.clear(); + + // start upload + this->UploadFolder(folder, rootPath.AsChar(), false); + } +} + +void BackendFileSystemWidget::ReplaceUuid(Core::Json& json, const char* internalName) +{ + auto jsonattribs = json.GetRootItem().Find("attributes"); + if (jsonattribs.IsArray()) + { + uint32 numattribs = jsonattribs.Size(); + for (uint32 i = 0; i < numattribs; i++) + { + auto jsoninternalname = jsonattribs[i].Find("internalName"); + auto jsonvalue = jsonattribs[i].Find("value"); + if (jsoninternalname.IsString() && jsonvalue.IsString()) + { + auto* sname = jsoninternalname.GetString(); + auto* olduuid = jsonvalue.GetString(); + if (std::string(internalName) == sname) + { + std::string newuuid = mLookup[olduuid]; + //qDebug() << "old uuid: " << olduuid << " new uuid: " << newuuid.c_str(); + if (!newuuid.empty()) + jsonvalue.SetString(newuuid.c_str()); + } + } + } + } } +QTreeWidgetItem* BackendFileSystemWidget::FindItemByPath(const QString& path, const QString& type) +{ + QStringList l = path.split("/"); + if (l.length() == 0) + return 0; + const int toplvls = mTreeWidget->topLevelItemCount(); + for (int i = 0; i < toplvls; i++) + { + QTreeWidgetItem* item = mTreeWidget->topLevelItem(i); + QString p = item->data(0, USERDATA_PATH).toString(); + if (p.length() && p[p.length()-1] == '/') + p.chop(1); + if (l[0] != p) + continue; + if (l.length() == 1 && type == "folder") + return item; + for (int j = 1; j < l.length(); j++) + { + bool found = false; + for (int k = 0; k < item->childCount(); k++) + { + auto* child = item->child(k); + QString childtype = child->data(0, USERDATA_TYPE).toString(); + QString wantedtype = j < l.length()-1 ? "folder" : type; + if (child->text(0) == l[j] && childtype == wantedtype) + { + item = item->child(k); + found = true; + break; + } + } + if (!found) + return 0; + } + return item; + } + return 0; +} + +void BackendFileSystemWidget::UploadFolder(const QString& plocal, const QString& pcloud, const bool xprun) +{ + QString pathlocal = plocal; + QString pathcloud = pcloud; + pathlocal.replace("//", "/"); + pathcloud.replace("//", "/"); + if (pathlocal.length() && pathlocal[pathlocal.length()-1] == '/') pathlocal.chop(1); + if (pathcloud.length() && pathcloud[pathcloud.length()-1] == '/') pathcloud.chop(1); + //qDebug() << "local: " << pathlocal << " cloud: " << pathcloud << " xprun: " << xprun; + + // iterate folders + QDirIterator dirit(pathlocal, QStringList(), QDir::Dirs); + while (dirit.hasNext()) + { + QString p = dirit.next(); + QFileInfo finf(p); + QString basename = finf.baseName(); + QString cloudfolder; + + // filter for "." and ".." + if (basename.isEmpty()) + continue; + + // build path on cloud + cloudfolder = pathcloud + '/' + basename; + cloudfolder.replace("//", "/"); + + // exists + if (auto* item = FindItemByPath(cloudfolder, "folder")) + UploadFolder(p, cloudfolder, xprun); + + // must be created + else if (auto* item = FindItemByPath(pathcloud, "folder")) + { + //qDebug() << "creating: " << pathcloud << " (" << "folder" << ")"; + SelectionItem model = CreateSelectionItem(item); + if (!model.GetCreud().Create()) + continue; + mPendingUploads++; + FoldersCreateRequest request(GetUser()->GetToken(), basename.toLatin1().constData(), model.GetUuid()); + QNetworkReply* reply = GetBackendInterface()->GetNetworkAccessManager()->ProcessRequest( request ); + connect(reply, &QNetworkReply::finished, this, [reply, this, p, cloudfolder, xprun]() + { + QNetworkReply* networkReply = qobject_cast( sender() ); + FoldersCreateResponse response(networkReply); + mPendingUploads--; + if (!response.HasError()) + this->Refresh(p, cloudfolder, xprun); + if (mPendingUploads == 0 && !xprun) { + UploadFolder(mLocalUploadRoot, mCloudUploadRoot, true); + } + }); + } + else + assert(false); + } + + // filter for types + QStringList filter; + if (xprun) { + filter << "*.xp.json"; // experience + } + else { + filter << "*.cs.json"; // classifier + filter << "*.sm.json"; // statemachine + } + + // iterate files + QDirIterator filesit(pathlocal, filter, QDir::Files); + while (filesit.hasNext()) + { + QString f = filesit.next(); + + // load file + QFile file(f); + if (!file.open(QFile::ReadOnly | QFile::Text)) { + QMessageBox::warning(this, "Error", "Failed to open file", QMessageBox::Ok); + continue; + } + QTextStream in(&file); + QByteArray arr(file.readAll()); + + // verify json + Json json; + if (!json.Parse(arr.constData())) { + QMessageBox::warning(this, "Error", "Failed to parse JSON", QMessageBox::Ok); + continue; + } + + // get file extension and map to type + QFileInfo finf(f); + QString suff = finf.completeSuffix(); + QString basename = finf.baseName(); + QString type = + suff == "cs.json" ? "CLASSIFIER" : + suff == "sm.json" ? "STATEMACHINE" : + suff == "xp.json" ? "EXPERIENCE" : "folder"; + + // remove file extension with . + f.chop(suff.length()+1); + + // build path on cloud + f.remove(pathlocal); + f = pathcloud + '/' + f; + f.replace("//", "/"); + + // update existing file + if (auto* item = FindItemByPath(f, type)) + { + //qDebug() << "updating: " << f << " (" << type << ")"; + SelectionItem fileModel = CreateSelectionItem(item); + if (!fileModel.GetCreud().Update()) + continue; + + // try to load old uuid, store in map and replace with new + auto jsonuuid = json.GetRootItem().Find("uuid"); + if (jsonuuid.IsString()) + { + mLookup[jsonuuid.GetString()] = fileModel.GetUuid(); + jsonuuid.SetString(fileModel.GetUuid()); + } + + // replace linked uuids for classifier and statemachine if known + if (type == "EXPERIENCE") + { + ReplaceUuid(json, "classifierUuid"); + ReplaceUuid(json, "stateMachineUuid"); + } + + // serialize + Core::String jsonstr; + json.WriteToString(jsonstr, true); + + // start update request + mPendingUploads++; + FilesUpdateRequest request(GetUser()->GetToken(), fileModel.GetUuid(), jsonstr.AsChar()); + QNetworkReply* reply = GetBackendInterface()->GetNetworkAccessManager()->ProcessRequest(request); + connect(reply, &QNetworkReply::finished, this, [reply, this, xprun]() + { + QNetworkReply* networkReply = qobject_cast(sender()); + FilesUpdateResponse response(networkReply); + mPendingUploads--; + if (mPendingUploads == 0) { + if (xprun) this->Refresh(); + else UploadFolder(mLocalUploadRoot, mCloudUploadRoot, true); + } + if (response.HasError()) + { + QMessageBox::warning(this, "Error", "Upload failed", QMessageBox::Ok); + return; + } + }); + } + + // create new file + else if (auto* item = FindItemByPath(pathcloud, "folder")) + { + //qDebug() << "creating: " << f << " (" << type << ")"; + SelectionItem folderModel = CreateSelectionItem(item); + if (!folderModel.GetCreud().Create()) + continue; + + // create empty file first + FilesCreateRequest request( + GetUser()->GetToken(), + basename.toLatin1().constData(), + folderModel.GetUuid(), + type.toLatin1().constData(), + "{}"); + + mPendingUploads++; + QNetworkReply* reply = GetBackendInterface()->GetNetworkAccessManager()->ProcessRequest(request); + connect(reply, &QNetworkReply::finished, this, [reply, this, json, xprun, type]() + { + QNetworkReply* networkReply = qobject_cast(sender()); + FilesCreateResponse response(networkReply); + mPendingUploads--; + + if (response.HasError()) + return; + + // try to load old uuid, store in map and replace with new + auto jsonitm = json.GetRootItem().Find("uuid"); + if (!jsonitm.IsNull()) + { + mLookup[jsonitm.GetString()] = response.mFileId; + jsonitm.SetString(response.mFileId); + } + + Json json2(json); + + // replace linked uuids for classifier and statemachine if known + if (type == "EXPERIENCE") + { + ReplaceUuid(json2, "classifierUuid"); + ReplaceUuid(json2, "stateMachineUuid"); + } + + // serialize + Core::String jsonstr; + json2.WriteToString(jsonstr, true); + + // update json content on backend + mPendingUploads++; + FilesUpdateRequest request(GetUser()->GetToken(), response.mFileId, jsonstr.AsChar()); + QNetworkReply* reply = GetBackendInterface()->GetNetworkAccessManager()->ProcessRequest(request); + connect(reply, &QNetworkReply::finished, this, [reply, this, xprun]() + { + QNetworkReply* networkReply = qobject_cast(sender()); + FilesUpdateResponse response(networkReply); + mPendingUploads--; + if (mPendingUploads == 0) { + if (xprun) this->Refresh(); + else UploadFolder(mLocalUploadRoot, mCloudUploadRoot, true); + } + if (response.HasError()) + { + QMessageBox::warning(this, "Error", "Upload failed", QMessageBox::Ok); + return; + } + }); + }); + } + else + assert(false); + } + //qDebug() << "Pending: " << std::to_string(mPendingUploads).c_str(); +} // called when the minus button got clicked void BackendFileSystemWidget::OnRemoveItem() diff --git a/src/Studio/Plugins/BackendFileSystem/BackendFileSystemWidget.h b/src/Studio/Plugins/BackendFileSystem/BackendFileSystemWidget.h index 41bbf6293..b47c8543f 100644 --- a/src/Studio/Plugins/BackendFileSystem/BackendFileSystemWidget.h +++ b/src/Studio/Plugins/BackendFileSystem/BackendFileSystemWidget.h @@ -155,7 +155,7 @@ class BackendFileSystemWidget : public QWidget bool ExpandByPath(const QStringList& itemPath); public slots: - void Refresh(); + void Refresh(const QString& localfolder = "", const QString& cloudfolder = "", const bool xprun = false); void OnCreateFile(); void OnCreateFolder(); void OnSearchFieldTextEdited(const QString & text); @@ -167,7 +167,7 @@ class BackendFileSystemWidget : public QWidget void OnCopyFileToPersonalFolder(); void OnContextMenuRetrieveItemRevision(); - void OnLoadFromDiskAndSaveToCloud(); + void OnUploadFromDisk(); void OnCopyJsonToClipboard(); void OnSaveToDisk(); @@ -237,6 +237,10 @@ class BackendFileSystemWidget : public QWidget CollapseState* FindCollapsedState(const char* uuid); bool IsItemCollapsed(const char* uuid); + void ReplaceUuid(Core::Json& json, const char* internalName); + QTreeWidgetItem* FindItemByPath(const QString& path, const QString& type); + void UploadFolder(const QString& pathlocal, const QString& pathcloud, const bool xprun); + Core::Array mSelectedItems; Core::Array mFolderCollapseStates; @@ -256,6 +260,10 @@ class BackendFileSystemWidget : public QWidget ImageButton* mRefreshButton; SearchBoxWidget* mSearchBox; + size_t mPendingUploads; + std::map mLookup; + QString mLocalUploadRoot; + QString mCloudUploadRoot; };