Support for tweaking concurrency parameters to not overwhelm low RU c…

…ontainers
wivuu · Dec 4, 2020 · 99f47aa · 99f47aa
1 parent 38f6da1
commit 99f47aa
Show file tree

Hide file tree

Showing 5 changed files with 58 additions and 15 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -10,7 +10,14 @@
             "request": "launch",
             "preLaunchTask": "build",
             "program": "${workspaceFolder}/bin/Debug/net5.0/AzCosmosCopy.dll",
-            "args": [],
+            "args": [
+                "-s", "${input:source}",
+                "--sd", "${input:sourcedb}",
+                "-d", "${input:dest} ",
+                "--dd", "${input:destdb}",
+                "--pc", "1",
+                "--pd", "5"
+            ],
             "cwd": "${workspaceFolder}",
             "console": "internalConsole",
             "stopAtEntry": false
@@ -21,5 +28,27 @@
             "request": "attach",
             "processId": "${command:pickProcess}"
         }
+    ],
+    "inputs": [
+        {
+            "id": "source",
+            "description": "Source account connection string",
+            "type": "promptString"
+        },
+        {
+            "id": "sourcedb",
+            "description": "Source account database",
+            "type": "promptString"
+        },
+        {
+            "id": "dest",
+            "description": "Destination account connection string",
+            "type": "promptString"
+        },
+        {
+            "id": "destdb",
+            "description": "Destination account database",
+            "type": "promptString"
+        },
     ]
 }
diff --git a/DbCopier.cs b/DbCopier.cs
@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
-using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Text.Json;
 using System.Text.Json.Serialization;
@@ -211,7 +210,7 @@ async Task CopyDataPipeline(ChannelWriter<CopyDiagnostic> channel)
 
             var buffer = new BufferBlock<ContainerProperties>(new () 
             { 
-                BoundedCapacity   = options.MaxContainerBufferSize, 
+                BoundedCapacity   = Math.Max(options.MaxContainerBufferSize, options.MaxContainerParallel), 
                 CancellationToken = cancellationToken 
             });
 
@@ -284,7 +283,7 @@ Func<ContainerProperties, Task> CopyContainerFactory(ChannelWriter<CopyDiagnosti
 
                     var buffer = new BufferBlock<Document>(new () 
                     { 
-                        BoundedCapacity   = options.MaxDocCopyBufferSize, 
+                        BoundedCapacity   = Math.Max(options.MaxDocCopyBufferSize, options.MaxDocCopyParallel), 
                         CancellationToken = cancellationToken 
                     });
 

diff --git a/Program.cs b/Program.cs
@@ -24,6 +24,14 @@
 
     new Option(
         new [] { "-m", "--minimal" }, "Output minimal information"
+    ),
+
+    new Option<int?>(
+        new [] { "--pc", "--parallel-containers" }, "Parallel container copies"
+    ),
+
+    new Option<int?>(
+        new [] { "--pd", "--parallel-documents" }, "Parallel document copies"
     )
 };
 
@@ -57,14 +65,21 @@
         };
 
         var sourceClient = new CosmosClient(args.Source, dbOptions);
-        var destClient   = new CosmosClient(args.Destination ?? args.Source, dbOptions);
+
+        var destClient = new CosmosClient(
+            string.IsNullOrWhiteSpace(args.Destination) ? args.Source : args.Destination, 
+            dbOptions);
 
         var copyOptions = new DbCopierOptions(
             sourceClient,
             destClient,
             args.SourceDatabase,
             args.DestinationDatabase ?? args.SourceDatabase
-        );
+        )
+        {
+            MaxContainerParallel = args.ParallelContainers,
+            MaxDocCopyParallel   = args.ParallelDocuments,
+        };
 
         var result = args.Minimal
             ? await DbCopier.CopyMinimal(copyOptions, cancellation.Token)
@@ -89,4 +104,6 @@ class Args
     public string? Destination { get; init; }
     public string? DestinationDatabase { get; init; }
     public bool Minimal { get; set; }
+    public int ParallelContainers { get; set; } = 10;
+    public int ParallelDocuments { get; set; } = 100;
 }
diff --git a/README.md b/README.md
@@ -23,14 +23,12 @@ Command line
 ```
 Options:
   -s, --source <source>                                  Source connection string (required)
-  --sd, --source-database <source-database>              Source database name (required)    
-  -d, --destination <destination>                        Destination connection string      
+  --sd, --source-database <source-database>              Source database name (required)
+  -d, --destination <destination>                        Destination connection string
   --dd, --destination-database <destination-database>    Destination database name
   -m, --minimal                                          Output minimal information
+  --parallel-containers, --pc <parallel-containers>      Parallel container copies
+  --parallel-documents, --pd <parallel-documents>        Parallel document copies
   --version                                              Show version information
-  -?, -h, --help                                         Show help and usage information   
+  -?, -h, --help                                         Show help and usage information
 ```
-
-
-## Coming soon
-- Support for tweaking concurrency parameters to not overwhelm low RU containers
diff --git a/Wivuu.AzCosmosCopy.csproj b/Wivuu.AzCosmosCopy.csproj
@@ -12,13 +12,13 @@
     <ToolCommandName>AzCosmosCopy</ToolCommandName>
     <PackageOutputPath>./nupkg</PackageOutputPath>
 
-    <VersionPrefix>0.0.4</VersionPrefix>
+    <VersionPrefix>1.0.0</VersionPrefix>
     <Authors>Erik O'Leary</Authors>
     <Company>Wivuu</Company>
     <Product>Wivuu AzCosmosCopy</Product>
     <Description>Copy cosmos db databases.</Description>
     <ReleaseNotes>
-    - Handle errors and cancellation in tui
+    - Support for tweaking concurrency parameters to not overwhelm low RU containers
     </ReleaseNotes>
     <PackageTags>cosmos;cosmosdb;azure;copy</PackageTags>
     <PackageProjectUrl>https://github.com/wivuu/Wivuu.AzCosmosCopy</PackageProjectUrl>