Allow minimum chunk size to be specified

barman/clients/cloud_backup.py

@@ -158,6 +158,7 @@ def main(args=None):
                 "server_name": config.server_name,
                 "compression": config.compression,
                 "max_archive_size": config.max_archive_size,
+                "min_chunk_size": config.min_chunk_size,
                 "cloud_interface": cloud_interface,
             }
             if __is_hook_script():
@@ -313,6 +314,14 @@ def parse_arguments(args=None):
         "(default: 100GB)",
         default="100GB",
     )
+    parser.add_argument(
+        "--min-chunk-size",
+        type=check_size,
+        help="minimum size of an individual chunk when uploading to cloud storage "
+        "(default: 5MB for aws-s3, 64KB for azure-blob-storage, not applicable for "
+        "google-cloud-storage)",
+        default=None,  # Defer to the cloud interface if nothing is specified
+    )
     parser.add_argument(
         "-d",
         "--dbname",

barman/cloud.py

@@ -253,14 +253,22 @@ def close(self):
 
 
 class CloudUploadController(object):
-    def __init__(self, cloud_interface, key_prefix, max_archive_size, compression):
+    def __init__(
+        self,
+        cloud_interface,
+        key_prefix,
+        max_archive_size,
+        compression,
+        min_chunk_size=None,
+    ):
         """
         Create a new controller that upload the backup in cloud storage
 
         :param CloudInterface cloud_interface: cloud interface instance
         :param str|None key_prefix: path inside the bucket
         :param int max_archive_size: the maximum size of an archive
         :param str|None compression: required compression
+        :param int|None min_chunk_size: the minimum size of a single upload part
         """
 
         self.cloud_interface = cloud_interface
@@ -275,10 +283,17 @@ def __init__(self, cloud_interface, key_prefix, max_archive_size, compression):
                 pretty_size(self.cloud_interface.MAX_ARCHIVE_SIZE),
             )
             self.max_archive_size = self.cloud_interface.MAX_ARCHIVE_SIZE
-        # We aim to a maximum of MAX_CHUNKS_PER_FILE / 2 chinks per file
-        self.chunk_size = 2 * int(
+        # We aim to a maximum of MAX_CHUNKS_PER_FILE / 2 chunks per file
+        calculated_chunk_size = 2 * int(
             max_archive_size / self.cloud_interface.MAX_CHUNKS_PER_FILE
         )
+        # Use whichever is higher - the calculated chunk_size or the requested
+        # min_chunk_size. Note that if the cloud interface has a MIN_CHUNK_SIZE
+        # higher than CloudUploadController.chunk_size then it will be used instead.
+        if min_chunk_size is not None:
+            self.chunk_size = max(calculated_chunk_size, min_chunk_size)
+        else:
+            self.chunk_size = calculated_chunk_size
         self.compression = compression
         self.tar_list = {}
 
@@ -1469,6 +1484,7 @@ def __init__(
         postgres,
         compression=None,
         backup_name=None,
+        min_chunk_size=None,
     ):
         """
         Base constructor.
@@ -1482,6 +1498,7 @@ def __init__(
         :param str compression: Compression algorithm to use
         :param str|None backup_name: A friendly name which can be used to reference
             this backup in the future.
+        :param int min_chunk_size: the minimum size of a single upload part
         """
         super(CloudBackupUploader, self).__init__(
             server_name,
@@ -1492,6 +1509,7 @@ def __init__(
 
         self.compression = compression
         self.max_archive_size = max_archive_size
+        self.min_chunk_size = min_chunk_size
 
         # Object properties set at backup time
         self.controller = None
@@ -1531,6 +1549,7 @@ def _create_upload_controller(self, backup_id):
             key_prefix,
             self.max_archive_size,
             self.compression,
+            self.min_chunk_size,
         )
 
     def _backup_data_files(