Update azure multipart policy

quickwit-oss · Nov 18, 2024 · 6955db5 · 6955db5
1 parent fe0f39a
commit 6955db5
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs
@@ -107,7 +107,15 @@ impl AzureBlobStorage {
             container_client,
             uri,
             prefix: PathBuf::new(),
-            multipart_policy: MultiPartPolicy::default(),
+            multipart_policy: MultiPartPolicy {
+                // Azure max part size is 100MB
+                // https://azure.microsoft.com/en-us/blog/general-availability-larger-block-blobs-in-azure-storage/
+                target_part_num_bytes: 100_000_000,
+                multipart_threshold_num_bytes: 100_000_000,
+                max_num_parts: 50_000, // Azure allows up to 50,000 blocks
+                max_object_num_bytes: 4_770_000_000_000u64, // Azure allows up to 4.77TB objects
+                max_concurrent_uploads: 100,
+            },
             retry_params: RetryParams::aggressive(),
         }
     }