From 35c7020d99a4d5d2ffa4b3f32158128f9b109978 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Mon, 20 Jan 2025 10:33:06 +0800 Subject: [PATCH 01/12] Add maxChunkSize getter function to get chunked upload max size from capabilities This will be added with NC31 Signed-off-by: Claudio Cambra --- src/libsync/capabilities.cpp | 5 +++++ src/libsync/capabilities.h | 1 + 2 files changed, 6 insertions(+) diff --git a/src/libsync/capabilities.cpp b/src/libsync/capabilities.cpp index f7b84716f2daa..e52d94e1ebffc 100644 --- a/src/libsync/capabilities.cpp +++ b/src/libsync/capabilities.cpp @@ -245,6 +245,11 @@ bool Capabilities::chunkingNg() const return _capabilities["dav"].toMap()["chunking"].toByteArray() >= "1.0"; } +qint64 Capabilities::maxChunkSize() const +{ + return _capabilities["files"].toMap()["chunked_upload"].toMap()["max_size"].toLongLong(); +} + bool Capabilities::bulkUpload() const { return _capabilities["dav"].toMap()["bulkupload"].toByteArray() >= "1.0"; diff --git a/src/libsync/capabilities.h b/src/libsync/capabilities.h index da10854528b30..5e0b0bcfd4d42 100644 --- a/src/libsync/capabilities.h +++ b/src/libsync/capabilities.h @@ -64,6 +64,7 @@ class OWNCLOUDSYNC_EXPORT Capabilities [[nodiscard]] bool shareResharing() const; [[nodiscard]] int shareDefaultPermissions() const; [[nodiscard]] bool chunkingNg() const; + [[nodiscard]] qint64 maxChunkSize() const; [[nodiscard]] bool bulkUpload() const; [[nodiscard]] bool filesLockAvailable() const; [[nodiscard]] bool filesLockTypeAvailable() const; From d7c5435e4277e25569792f917464a6bc66927075 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Mon, 20 Jan 2025 10:34:17 +0800 Subject: [PATCH 02/12] Override config chunk sizing values with max chunk size retrieved from capabilities Signed-off-by: Claudio Cambra --- src/gui/folder.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index bcb3ed70f7547..7e30d02f3ab6e 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -1144,18 +1144,24 @@ SyncOptions Folder::initializeSyncOptions() const { SyncOptions opt; ConfigFile cfgFile; + const auto account = _accountState->account(); auto newFolderLimit = cfgFile.newBigFolderSizeLimit(); opt._newBigFolderSizeLimit = newFolderLimit.first ? newFolderLimit.second * 1000LL * 1000LL : -1; // convert from MB to B opt._confirmExternalStorage = cfgFile.confirmExternalStorage(); opt._moveFilesToTrash = cfgFile.moveToTrash(); opt._vfs = _vfs; - opt._parallelNetworkJobs = _accountState->account()->isHttp2Supported() ? 20 : 6; + opt._parallelNetworkJobs = account->isHttp2Supported() ? 20 : 6; // Chunk V2: Size of chunks must be between 5MB and 5GB, except for the last chunk which can be smaller + const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); + const auto validCapsChunkSize = capsMaxChunkSize <= 0; + const auto maxChunkSize = validCapsChunkSize ? capsMaxChunkSize : cfgFile.maxChunkSize(); + const auto preferredChunkSize = validCapsChunkSize ? capsMaxChunkSize : cfgFile.chunkSize(); + opt.setMinChunkSize(cfgFile.minChunkSize()); - opt.setMaxChunkSize(cfgFile.maxChunkSize()); - opt._initialChunkSize = ::qBound(opt.minChunkSize(), cfgFile.chunkSize(), opt.maxChunkSize()); + opt.setMaxChunkSize(maxChunkSize); + opt._initialChunkSize = ::qBound(opt.minChunkSize(), preferredChunkSize, opt.maxChunkSize()); opt._targetChunkUploadDuration = cfgFile.targetChunkUploadDuration(); opt.fillFromEnvironmentVariables(); From a7d044f9e6c6d43cd497a2c47c05b2a2aa7b41ef Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Mon, 20 Jan 2025 11:55:33 +0800 Subject: [PATCH 03/12] Enforce chunk size provided by server capabilities Signed-off-by: Claudio Cambra --- src/gui/folder.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 7e30d02f3ab6e..ee028cdd248b4 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -1154,16 +1154,17 @@ SyncOptions Folder::initializeSyncOptions() const opt._parallelNetworkJobs = account->isHttp2Supported() ? 20 : 6; // Chunk V2: Size of chunks must be between 5MB and 5GB, except for the last chunk which can be smaller - const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); - const auto validCapsChunkSize = capsMaxChunkSize <= 0; - const auto maxChunkSize = validCapsChunkSize ? capsMaxChunkSize : cfgFile.maxChunkSize(); - const auto preferredChunkSize = validCapsChunkSize ? capsMaxChunkSize : cfgFile.chunkSize(); - - opt.setMinChunkSize(cfgFile.minChunkSize()); - opt.setMaxChunkSize(maxChunkSize); - opt._initialChunkSize = ::qBound(opt.minChunkSize(), preferredChunkSize, opt.maxChunkSize()); - opt._targetChunkUploadDuration = cfgFile.targetChunkUploadDuration(); - + if (const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); capsMaxChunkSize <= 0) { + opt.setMinChunkSize(capsMaxChunkSize); + opt.setMaxChunkSize(capsMaxChunkSize); + opt._initialChunkSize = capsMaxChunkSize); + } else { + const auto cfgMinChunkSize = cfgFile.minChunkSize(); + const auto cfgMaxChunkSize = cfgFile.maxChunkSize(); + opt.setMinChunkSize(cfgMinChunkSize); + opt.setMaxChunkSize(cfgMaxChunkSize); + opt._initialChunkSize = ::qBound(cfgMinChunkSize, cfgFile.chunkSize(), cfgMaxChunkSize); + } opt.fillFromEnvironmentVariables(); opt.verifyChunkSizes(); From 93570808d40d37efeed1607062224f55b594bd66 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Mon, 20 Jan 2025 11:59:27 +0800 Subject: [PATCH 04/12] Add config entry to override server chunk size Signed-off-by: Claudio Cambra --- src/gui/folder.cpp | 2 +- src/libsync/configfile.cpp | 7 +++++++ src/libsync/configfile.h | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index ee028cdd248b4..070f7575941b0 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -1154,7 +1154,7 @@ SyncOptions Folder::initializeSyncOptions() const opt._parallelNetworkJobs = account->isHttp2Supported() ? 20 : 6; // Chunk V2: Size of chunks must be between 5MB and 5GB, except for the last chunk which can be smaller - if (const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); capsMaxChunkSize <= 0) { + if (const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); capsMaxChunkSize <= 0 && !cfgFile.overrideServerChunkSize) { opt.setMinChunkSize(capsMaxChunkSize); opt.setMaxChunkSize(capsMaxChunkSize); opt._initialChunkSize = capsMaxChunkSize); diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp index 2b1cda432e597..1297e5067e50d 100644 --- a/src/libsync/configfile.cpp +++ b/src/libsync/configfile.cpp @@ -70,6 +70,7 @@ static constexpr char overrideLocalDirC[] = "overrideLocalDir"; static constexpr char isVfsEnabledC[] = "isVfsEnabled"; static constexpr char geometryC[] = "geometry"; static constexpr char timeoutC[] = "timeout"; +static constexpr char overrideServerChunkSize[] = "overrideServerChunkSize"; static constexpr char chunkSizeC[] = "chunkSize"; static constexpr char minChunkSizeC[] = "minChunkSize"; static constexpr char maxChunkSizeC[] = "maxChunkSize"; @@ -258,6 +259,12 @@ int ConfigFile::timeout() const return settings.value(QLatin1String(timeoutC), 300).toInt(); // default to 5 min } +bool ConfigFile::overrideServerChunkSize() const +{ + QSettings settings(configFile(), QSettings::IniFormat); + return settings.value(QLatin1String(overrideServerChunkSize), false).toBool(); +} + qint64 ConfigFile::chunkSize() const { QSettings settings(configFile(), QSettings::IniFormat); diff --git a/src/libsync/configfile.h b/src/libsync/configfile.h index 844036336a6f9..27a83a5c2e15c 100644 --- a/src/libsync/configfile.h +++ b/src/libsync/configfile.h @@ -177,6 +177,7 @@ class OWNCLOUDSYNC_EXPORT ConfigFile void setShowInExplorerNavigationPane(bool show); [[nodiscard]] int timeout() const; + [[nodiscard]] bool overrideServerChunkSize() const; [[nodiscard]] qint64 chunkSize() const; [[nodiscard]] qint64 maxChunkSize() const; [[nodiscard]] qint64 minChunkSize() const; From 36b1f92810bf7cd2c46818cb61f17b9f3ca67b55 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Mon, 20 Jan 2025 17:24:56 +0800 Subject: [PATCH 05/12] Fix naming for overrideSercerChunkSize constexpr Signed-off-by: Claudio Cambra --- src/libsync/configfile.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp index 1297e5067e50d..40b918a25681d 100644 --- a/src/libsync/configfile.cpp +++ b/src/libsync/configfile.cpp @@ -70,7 +70,7 @@ static constexpr char overrideLocalDirC[] = "overrideLocalDir"; static constexpr char isVfsEnabledC[] = "isVfsEnabled"; static constexpr char geometryC[] = "geometry"; static constexpr char timeoutC[] = "timeout"; -static constexpr char overrideServerChunkSize[] = "overrideServerChunkSize"; +static constexpr char overrideServerChunkSizeC[] = "overrideServerChunkSize"; static constexpr char chunkSizeC[] = "chunkSize"; static constexpr char minChunkSizeC[] = "minChunkSize"; static constexpr char maxChunkSizeC[] = "maxChunkSize"; @@ -262,7 +262,7 @@ int ConfigFile::timeout() const bool ConfigFile::overrideServerChunkSize() const { QSettings settings(configFile(), QSettings::IniFormat); - return settings.value(QLatin1String(overrideServerChunkSize), false).toBool(); + return settings.value(QLatin1String(overrideServerChunkSizeC), false).toBool(); } qint64 ConfigFile::chunkSize() const From 44a599e7ae668dcf9282b54852945f2a4eb4ebbf Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Mon, 20 Jan 2025 17:26:23 +0800 Subject: [PATCH 06/12] Remove extraneous parenthesis Signed-off-by: Claudio Cambra --- src/gui/folder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 070f7575941b0..44324bda2156e 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -1157,7 +1157,7 @@ SyncOptions Folder::initializeSyncOptions() const if (const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); capsMaxChunkSize <= 0 && !cfgFile.overrideServerChunkSize) { opt.setMinChunkSize(capsMaxChunkSize); opt.setMaxChunkSize(capsMaxChunkSize); - opt._initialChunkSize = capsMaxChunkSize); + opt._initialChunkSize = capsMaxChunkSize; } else { const auto cfgMinChunkSize = cfgFile.minChunkSize(); const auto cfgMaxChunkSize = cfgFile.maxChunkSize(); From a77f3271bdd19aa894e1c17cb27c8e62cc7ca334 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Mon, 20 Jan 2025 17:26:48 +0800 Subject: [PATCH 07/12] Fix capsMaxChunkSize comparison Signed-off-by: Claudio Cambra --- src/gui/folder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 44324bda2156e..50e82c7ec3b91 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -1154,7 +1154,7 @@ SyncOptions Folder::initializeSyncOptions() const opt._parallelNetworkJobs = account->isHttp2Supported() ? 20 : 6; // Chunk V2: Size of chunks must be between 5MB and 5GB, except for the last chunk which can be smaller - if (const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); capsMaxChunkSize <= 0 && !cfgFile.overrideServerChunkSize) { + if (const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); capsMaxChunkSize > 0 && !cfgFile.overrideServerChunkSize) { opt.setMinChunkSize(capsMaxChunkSize); opt.setMaxChunkSize(capsMaxChunkSize); opt._initialChunkSize = capsMaxChunkSize; From b3ff083c02842390732544b61bfc5a6d2cf945f3 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Mon, 20 Jan 2025 17:48:56 +0800 Subject: [PATCH 08/12] Do not apply max capabilities chunk size as min chunk size too Signed-off-by: Claudio Cambra --- src/gui/folder.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 50e82c7ec3b91..a39afee8912c7 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -1154,14 +1154,14 @@ SyncOptions Folder::initializeSyncOptions() const opt._parallelNetworkJobs = account->isHttp2Supported() ? 20 : 6; // Chunk V2: Size of chunks must be between 5MB and 5GB, except for the last chunk which can be smaller + const auto cfgMinChunkSize = cfgFile.minChunkSize(); + opt.setMinChunkSize(cfgMinChunkSize); + if (const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); capsMaxChunkSize > 0 && !cfgFile.overrideServerChunkSize) { - opt.setMinChunkSize(capsMaxChunkSize); opt.setMaxChunkSize(capsMaxChunkSize); opt._initialChunkSize = capsMaxChunkSize; } else { - const auto cfgMinChunkSize = cfgFile.minChunkSize(); const auto cfgMaxChunkSize = cfgFile.maxChunkSize(); - opt.setMinChunkSize(cfgMinChunkSize); opt.setMaxChunkSize(cfgMaxChunkSize); opt._initialChunkSize = ::qBound(cfgMinChunkSize, cfgFile.chunkSize(), cfgMaxChunkSize); } From 0f02d70e3cd3fd9ddb1aff8ccc190289dcd6dbe9 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Mon, 20 Jan 2025 17:49:12 +0800 Subject: [PATCH 09/12] Actually call overrideServerChunkSize Signed-off-by: Claudio Cambra --- src/gui/folder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index a39afee8912c7..5628ae7907233 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -1157,7 +1157,7 @@ SyncOptions Folder::initializeSyncOptions() const const auto cfgMinChunkSize = cfgFile.minChunkSize(); opt.setMinChunkSize(cfgMinChunkSize); - if (const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); capsMaxChunkSize > 0 && !cfgFile.overrideServerChunkSize) { + if (const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); capsMaxChunkSize > 0 && !cfgFile.overrideServerChunkSize()) { opt.setMaxChunkSize(capsMaxChunkSize); opt._initialChunkSize = capsMaxChunkSize; } else { From bca2f1db9d79ef51a8f43503e74ea273129d9fdd Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Mon, 20 Jan 2025 17:51:55 +0800 Subject: [PATCH 10/12] Remove overrideServerChunkSize config entry Signed-off-by: Claudio Cambra --- src/gui/folder.cpp | 2 +- src/libsync/configfile.cpp | 7 ------- src/libsync/configfile.h | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 5628ae7907233..54a08ec2e8f54 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -1157,7 +1157,7 @@ SyncOptions Folder::initializeSyncOptions() const const auto cfgMinChunkSize = cfgFile.minChunkSize(); opt.setMinChunkSize(cfgMinChunkSize); - if (const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); capsMaxChunkSize > 0 && !cfgFile.overrideServerChunkSize()) { + if (const auto capsMaxChunkSize = account->capabilities().maxChunkSize(); capsMaxChunkSize) { opt.setMaxChunkSize(capsMaxChunkSize); opt._initialChunkSize = capsMaxChunkSize; } else { diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp index 40b918a25681d..2b1cda432e597 100644 --- a/src/libsync/configfile.cpp +++ b/src/libsync/configfile.cpp @@ -70,7 +70,6 @@ static constexpr char overrideLocalDirC[] = "overrideLocalDir"; static constexpr char isVfsEnabledC[] = "isVfsEnabled"; static constexpr char geometryC[] = "geometry"; static constexpr char timeoutC[] = "timeout"; -static constexpr char overrideServerChunkSizeC[] = "overrideServerChunkSize"; static constexpr char chunkSizeC[] = "chunkSize"; static constexpr char minChunkSizeC[] = "minChunkSize"; static constexpr char maxChunkSizeC[] = "maxChunkSize"; @@ -259,12 +258,6 @@ int ConfigFile::timeout() const return settings.value(QLatin1String(timeoutC), 300).toInt(); // default to 5 min } -bool ConfigFile::overrideServerChunkSize() const -{ - QSettings settings(configFile(), QSettings::IniFormat); - return settings.value(QLatin1String(overrideServerChunkSizeC), false).toBool(); -} - qint64 ConfigFile::chunkSize() const { QSettings settings(configFile(), QSettings::IniFormat); diff --git a/src/libsync/configfile.h b/src/libsync/configfile.h index 27a83a5c2e15c..844036336a6f9 100644 --- a/src/libsync/configfile.h +++ b/src/libsync/configfile.h @@ -177,7 +177,6 @@ class OWNCLOUDSYNC_EXPORT ConfigFile void setShowInExplorerNavigationPane(bool show); [[nodiscard]] int timeout() const; - [[nodiscard]] bool overrideServerChunkSize() const; [[nodiscard]] qint64 chunkSize() const; [[nodiscard]] qint64 maxChunkSize() const; [[nodiscard]] qint64 minChunkSize() const; From 59f85467f759da10aef996aae17cbb3ca8deb81f Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Mon, 20 Jan 2025 17:56:28 +0800 Subject: [PATCH 11/12] Expose maxConcurrentChunkUploads from Capabilities Signed-off-by: Claudio Cambra --- src/libsync/capabilities.cpp | 5 +++++ src/libsync/capabilities.h | 1 + 2 files changed, 6 insertions(+) diff --git a/src/libsync/capabilities.cpp b/src/libsync/capabilities.cpp index e52d94e1ebffc..55e63874aa7ff 100644 --- a/src/libsync/capabilities.cpp +++ b/src/libsync/capabilities.cpp @@ -250,6 +250,11 @@ qint64 Capabilities::maxChunkSize() const return _capabilities["files"].toMap()["chunked_upload"].toMap()["max_size"].toLongLong(); } +int Capabilities::maxConcurrentChunkUploads() const +{ + return _capabilities["files"].toMap()["chunked_upload"].toMap()["max_parallel_count"].toInt(); +} + bool Capabilities::bulkUpload() const { return _capabilities["dav"].toMap()["bulkupload"].toByteArray() >= "1.0"; diff --git a/src/libsync/capabilities.h b/src/libsync/capabilities.h index 5e0b0bcfd4d42..b84fa8e7db223 100644 --- a/src/libsync/capabilities.h +++ b/src/libsync/capabilities.h @@ -65,6 +65,7 @@ class OWNCLOUDSYNC_EXPORT Capabilities [[nodiscard]] int shareDefaultPermissions() const; [[nodiscard]] bool chunkingNg() const; [[nodiscard]] qint64 maxChunkSize() const; + [[nodiscard]] int maxConcurrentChunkUploads() const; [[nodiscard]] bool bulkUpload() const; [[nodiscard]] bool filesLockAvailable() const; [[nodiscard]] bool filesLockTypeAvailable() const; From 46dc2a71f7691fc5e163759175e934e14376d496 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Mon, 20 Jan 2025 17:56:52 +0800 Subject: [PATCH 12/12] Apply max concurrent chunk uploads retrieved from server capabilities Signed-off-by: Claudio Cambra --- src/gui/folder.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 54a08ec2e8f54..d68a2196367f9 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -1151,7 +1151,11 @@ SyncOptions Folder::initializeSyncOptions() const opt._confirmExternalStorage = cfgFile.confirmExternalStorage(); opt._moveFilesToTrash = cfgFile.moveToTrash(); opt._vfs = _vfs; - opt._parallelNetworkJobs = account->isHttp2Supported() ? 20 : 6; + + const auto capsMaxConcurrentChunkUploads = account->capabilities().maxConcurrentChunkUploads(); + opt._parallelNetworkJobs = capsMaxConcurrentChunkUploads > 0 + ? capsMaxConcurrentChunkUploads + : account->isHttp2Supported() ? 20 : 6; // Chunk V2: Size of chunks must be between 5MB and 5GB, except for the last chunk which can be smaller const auto cfgMinChunkSize = cfgFile.minChunkSize();