Add support for headers with custom request object

gregyjames · Aug 5, 2024 · a9c5e2f · a9c5e2f
1 parent 568c2bd
commit a9c5e2f
Show file tree

Hide file tree

Showing 16 changed files with 146 additions and 94 deletions.
diff --git a/BenchmarkOctaneProject/Program.cs b/BenchmarkOctaneProject/Program.cs
@@ -78,13 +78,13 @@ public void GlobalSetup()
         [Benchmark]
         public async Task BenchmarkOctane()
         {
-            await _OctaneEngine.DownloadFile(Url, "output0.zip", pauseTokenSource, cancelTokenSource);
+            await _OctaneEngine.DownloadFile(new OctaneRequest(){URL = ""}, "output0.zip", pauseTokenSource, cancelTokenSource);
         }
 
         [Benchmark]
         public async Task BenchmarkOctaneLowMemory()
         {
-            await _OctaneEngine2.DownloadFile(Url, "output1.zip", pauseTokenSource, cancelTokenSource);
+            await _OctaneEngine2.DownloadFile(new OctaneRequest(){URL = ""}, "output1.zip", pauseTokenSource, cancelTokenSource);
         }
         [Benchmark]
         public async Task BenchmarkHttpClient()

diff --git a/OctaneEngine/Clients/DefaultClient.cs b/OctaneEngine/Clients/DefaultClient.cs
@@ -75,6 +75,14 @@ public void SetArrayPool(ArrayPool<byte> pool)
         _memPool = pool;
     }
 
+    public void SetHeaders(OctaneRequest req)
+    {
+        foreach (var pair in req.Headers)
+        {
+            _httpClient.DefaultRequestHeaders.Add(pair.Key, pair.Value);
+        }
+    }
+
     private async PooledTask CopyMessageContentToStreamWithProgressAsync(HttpResponseMessage message, Stream stream, IProgress<long> progress)
     {
         byte[] buffer = _memPool.Rent(_config.BufferSize);

diff --git a/OctaneEngine/Clients/HTTPClientModule.cs b/OctaneEngine/Clients/HTTPClientModule.cs
@@ -30,7 +30,7 @@ protected override void Load(ContainerBuilder builder)
             {
                 MaxResponseContentBufferSize = cfg.BufferSize,
             };
-
+            
             return _client;
         }).As<HttpClient>().SingleInstance();
     }

diff --git a/OctaneEngine/Clients/IClient.cs b/OctaneEngine/Clients/IClient.cs
@@ -39,5 +39,6 @@ public interface IClient : IDisposable
     public void SetMmf(MemoryMappedFile file);
     public void SetProgressbar(ProgressBar bar);
     public void SetArrayPool(ArrayPool<Byte> pool);
+    public void SetHeaders(OctaneRequest req);
     public PooledTask Download(string url, (long, long) piece, CancellationToken cancellationToken, PauseToken pauseToken);
 }
diff --git a/OctaneEngine/Clients/OctaneClient.cs b/OctaneEngine/Clients/OctaneClient.cs
@@ -80,7 +80,15 @@ public void SetArrayPool(ArrayPool<byte> pool)
     {
         _memPool = pool;
     }
-
+
+    public void SetHeaders(OctaneRequest req)
+    {
+        foreach (var pair in req.Headers)
+        {
+            _client.DefaultRequestHeaders.Add(pair.Key, pair.Value);
+        }
+    }
+
     public async PooledTask Download(string url,(long, long) piece, CancellationToken cancellationToken, PauseToken pauseToken)
     {
         _log.LogTrace("Sending request for range ({PieceItem1},{PieceItem2})...", piece.Item1, piece.Item2);

diff --git a/OctaneEngine/Engine.cs b/OctaneEngine/Engine.cs
@@ -136,13 +136,13 @@ public void SetProxy(IWebProxy proxy)
         /// <param name="outFile">The output file name of the download. Use 'null' to get file name from url.</param>
         /// <param name="pauseTokenSource">The pause token source to use for pausing and resuming.</param>
         /// <param name="cancelTokenSource">The cancellation token for canceling the task.</param>
-        public async Task DownloadFile(string url, string outFile = null, PauseTokenSource pauseTokenSource = null, CancellationTokenSource cancelTokenSource = null)
+        public async Task DownloadFile(OctaneRequest req, string outFile = null, PauseTokenSource pauseTokenSource = null, CancellationTokenSource cancelTokenSource = null)
         {
-            var (_length, _range) = await getFileSizeAndRangeSupport(url);
+            var (_length, _range) = await getFileSizeAndRangeSupport(req.URL);
 
             #region Varible Initilization
                 var logger = _factory.CreateLogger("OctaneEngine");
-                var filename = outFile ?? Path.GetFileName(new Uri(url).LocalPath);
+                var filename = outFile ?? Path.GetFileName(new Uri(req.URL).LocalPath);
                 var success = false;
                 var cancellation_token = Helpers.CreateCancellationToken(cancelTokenSource, _config, filename);
                 var pause_token = pauseTokenSource ?? new PauseTokenSource(_factory);
@@ -168,7 +168,7 @@ public async Task DownloadFile(string url, string outFile = null, PauseTokenSour
             logger.LogInformation($"PART SIZE: {NetworkAnalyzer.PrettySize(partSize)}");
 
             stopwatch.Start();
-            _client.SetBaseAddress(url);
+            _client.SetBaseAddress(req.URL);
 
             try
             {
@@ -178,6 +178,7 @@ public async Task DownloadFile(string url, string outFile = null, PauseTokenSour
                     if (_client.IsRangeSupported())
                     {
                         var pieces = Helpers.CreatePartsList(_length, partSize, logger);
+                        _client.SetHeaders(req);
                         _client.SetMmf(mmf);
                         _client.SetArrayPool(memPool);
                         logger.LogInformation("Using Octane Client to download file.");
@@ -198,7 +199,7 @@ public async Task DownloadFile(string url, string outFile = null, PauseTokenSour
                         {
                             await Parallel.ForEachAsync(pieces, options, async (piece, token) =>
                             {
-                                await _client.Download(url, piece, cancellation_token, pause_token.Token);
+                                await _client.Download(req.URL, piece, cancellation_token, pause_token.Token);
 
                                 Interlocked.Increment(ref tasksDone);
 
@@ -221,7 +222,8 @@ await Parallel.ForEachAsync(pieces, options, async (piece, token) =>
                     else
                     {
                         logger.LogInformation("Using Default Client to download file.");
-                        await _normalClient.Download(url, (0, 0), cancellation_token, pause_token.Token);
+                        _normalClient.SetHeaders(req);
+                        await _normalClient.Download(req.URL, (0, 0), cancellation_token, pause_token.Token);
                     }
 
                     success = true;

diff --git a/OctaneEngine/IEngine.cs b/OctaneEngine/IEngine.cs
@@ -7,7 +7,7 @@ namespace OctaneEngineCore;
 
 public interface IEngine
 {
-    public Task DownloadFile(string url, string outFile = null, PauseTokenSource pauseTokenSource = null, CancellationTokenSource cancelTokenSource = null);
+    public Task DownloadFile(OctaneRequest octaneRequest, string outFile = null, PauseTokenSource pauseTokenSource = null, CancellationTokenSource cancelTokenSource = null);
     public void SetProgressCallback(Action<double> callback);
     public void SetDoneCallback(Action<bool> callback);
     public void SetProxy(IWebProxy proxy);

diff --git a/OctaneEngine/OctaneRequest.cs b/OctaneEngine/OctaneRequest.cs
@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Net;
+
+namespace OctaneEngineCore;
+
+public class OctaneRequest
+{
+    public Dictionary<string, string> Headers { get; set; }
+    public string URL { get; set; }
+
+    public OctaneRequest()
+    {
+        Headers = new Dictionary<string, string>();
+    }
+}
diff --git a/OctaneEngine/docs/README.md b/OctaneEngine/docs/README.md
@@ -24,39 +24,49 @@ dotnet add package OctaneEngineCore
 
 # Usage
 ```csharp
-private const string Url = "https://plugins.jetbrains.com/files/7973/281233/sonarlint-intellij-7.4.0.60471.zip?updateId=281233&pluginId=7973&family=INTELLIJ";
-private static void Main()
-{
-     //Logging Setup
-     var seriLog = new LoggerConfiguration()
-          .Enrich.FromLogContext()
-          .MinimumLevel.Verbose()
-          .WriteTo.File("./OctaneLog.txt")
-          .WriteTo.Console(theme: AnsiConsoleTheme.Sixteen)
-          .CreateLogger();
-     var factory = LoggerFactory.Create(logging => {
-          logging.AddSerilog(seriLog);
-     });
-
-     //JSON Config Loading
-     var builder = new ConfigurationBuilder();
-     builder.SetBasePath(Directory.GetCurrentDirectory())
-          .AddJsonFile("appsettings.json", true, true);
-     var configRoot = builder.Build();
-     var config = new OctaneConfiguration(configRoot, factory);
-
-     //Find Optimal number of parts
-     var optimalNumberOfParts = Engine.GetOptimalNumberOfParts(Url).Result;
-     seriLog.Information("Optimal number of parts to download file: {OptimalNumberOfParts}", optimalNumberOfParts);
-
-     seriLog.Information("Speed: {Result}", NetworkAnalyzer.GetCurrentNetworkSpeed().Result);
-     seriLog.Information("Latency: {Result}", NetworkAnalyzer.GetCurrentNetworkLatency().Result);
-     var pauseTokenSource = new PauseTokenSource();
-     var cancelTokenSource = new CancellationTokenSource();
+#region Logging Configuration
+var seriLog = new LoggerConfiguration()
+    .Enrich.FromLogContext()
+    .MinimumLevel.Error()
+    .WriteTo.Async(a => a.File("./OctaneLog.txt"))
+    .WriteTo.Async(a => a.Console(theme: AnsiConsoleTheme.Sixteen))
+    .CreateLogger();
+    var factory = LoggerFactory.Create(logging => {
+        logging.AddSerilog(seriLog);
+    });
+#endregion
+
+#region Configuration Loading
+    var builder = new ConfigurationBuilder();
+    builder.SetBasePath(Directory.GetCurrentDirectory())
+        .AddJsonFile("appsettings.json", true, true);
+    var configRoot = builder.Build();
+#endregion
+
+#region Find and Set optimal number of parts
+    //var optimalNumberOfParts = Engine.GetOptimalNumberOfParts(Url).Result;
+    //seriLog.Information("Optimal number of parts to download file: {OptimalNumberOfParts}", optimalNumberOfParts);
+#endregion
 
-     var octaneEngine = new Engine(factory, config);
-     octaneEngine.DownloadFile(Url, null, pauseTokenSource, cancelTokenSource).Wait(cancelTokenSource.Token);
-
+//seriLog.Information("Speed: {Result}", NetworkAnalyzer.GetCurrentNetworkSpeed().Result);
+//seriLog.Information("Latency: {Result}", NetworkAnalyzer.GetCurrentNetworkLatency().Result);
+var pauseTokenSource = new PauseTokenSource();
+using var cancelTokenSource = new CancellationTokenSource();
+var containerBuilder = new ContainerBuilder();
+containerBuilder.RegisterInstance(factory).As<ILoggerFactory>();
+containerBuilder.RegisterInstance(configRoot).As<IConfiguration>();
+containerBuilder.AddOctane();
+var engineContainer = containerBuilder.Build();
+var engine = engineContainer.Resolve<IEngine>();
+engine.DownloadFile(
+    new OctaneRequest(){
+        URL = Url,
+        Headers = new Dictionary<string, string>(){}
+    }, 
+    null, 
+    pauseTokenSource, 
+    cancelTokenSource
+).Wait();
 ```
 
 # Benchmark

diff --git a/OctaneTestProject/DownloadTest.cs b/OctaneTestProject/DownloadTest.cs
@@ -85,7 +85,7 @@ public void DownloadFile()
                     Assert.IsTrue(File.Exists(_outFile));
                 });
                 engine.SetProgressCallback(Console.WriteLine);
-                engine.DownloadFile(url, _outFile, _pauseTokenSource, _cancelTokenSource).Wait();
+                engine.DownloadFile(new OctaneRequest(){URL = url}, _outFile, _pauseTokenSource, _cancelTokenSource).Wait();
             }
             catch
             {

diff --git a/OctaneTestProject/EqualityTest.cs b/OctaneTestProject/EqualityTest.cs
@@ -92,7 +92,7 @@ public void FileEqualityTest()
                     engine.SetProgressCallback(Console.WriteLine);
                     engine.SetProxy(null);
 
-                    var t = engine.DownloadFile(url, outFile, _pauseTokenSource, _cancelTokenSource);
+                    var t = engine.DownloadFile(new OctaneRequest(){URL = url}, outFile, _pauseTokenSource, _cancelTokenSource);
                     t.Wait();
                 }
             }

diff --git a/OctaneTestProject/PauseResumeTest.cs b/OctaneTestProject/PauseResumeTest.cs
@@ -78,7 +78,7 @@ public void PauseResumeFile()
             engine.SetProxy(null);
             Parallel.Invoke(
                 () => Action(_pauseTokenSource),
-                () => engine.DownloadFile(url, _outFile, _pauseTokenSource, _cancelTokenSource).Wait()
+                () => engine.DownloadFile(new OctaneRequest(){URL = url}, _outFile, _pauseTokenSource, _cancelTokenSource).Wait()
             );
         }
 

diff --git a/OctaneTester/Examples/Autofac.cs b/OctaneTester/Examples/Autofac.cs
@@ -38,6 +38,6 @@ public void AutofacExample()
         containerBuilder.AddOctane();
         var engineContainer = containerBuilder.Build();
         var engine = engineContainer.Resolve<IEngine>();
-        engine.DownloadFile("", "", pauseTokenSource, cancelTokenSource);
+        engine.DownloadFile(new OctaneRequest(){URL = ""}, "", pauseTokenSource, cancelTokenSource);
     }
 }
diff --git a/OctaneTester/Examples/NoLogger.cs b/OctaneTester/Examples/NoLogger.cs
@@ -27,6 +27,6 @@ public void NoLoggerExample()
         containerBuilder.AddOctane();
         var engineContainer = containerBuilder.Build();
         var engine = engineContainer.Resolve<IEngine>();
-        engine.DownloadFile("", "", pauseTokenSource, cancelTokenSource);
+        engine.DownloadFile(new OctaneRequest(){URL = ""}, "", pauseTokenSource, cancelTokenSource);
     }
 }
diff --git a/OctaneTester/Program.cs b/OctaneTester/Program.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Threading;
 using Autofac;
@@ -52,7 +53,14 @@ private static void Main()
             containerBuilder.AddOctane();
             var engineContainer = containerBuilder.Build();
             var engine = engineContainer.Resolve<IEngine>();
-            engine.DownloadFile(Url, null, pauseTokenSource, cancelTokenSource).Wait();
+            engine.DownloadFile(new OctaneRequest()
+            {
+                URL = Url,
+                Headers = new Dictionary<string, string>()
+                {
+
+                }
+            }, null, pauseTokenSource, cancelTokenSource).Wait();
         }
     }
 }
diff --git a/README.md b/README.md
@@ -1,11 +1,8 @@
 ![C#](https://github.com/gregyjames/OctaneDownloader/actions/workflows/dotnet.yml/badge.svg)
 [![CodeQL](https://github.com/gregyjames/OctaneDownloader/actions/workflows/codeql-analysis.yml/badge.svg?branch=master)](https://github.com/gregyjames/OctaneDownloader/actions/workflows/codeql-analysis.yml)
 [![CodeFactor](https://www.codefactor.io/repository/github/gregyjames/octanedownloader/badge)](https://www.codefactor.io/repository/github/gregyjames/octanedownloader)
-[![codebeat badge](https://codebeat.co/badges/9154fd6f-ac4b-4f00-8910-66488582efcd)](https://codebeat.co/projects/github-com-gregyjames-octanedownloader-master)
 [![NuGet latest version](https://badgen.net/nuget/v/OctaneEngineCore)](https://www.nuget.org/packages/OctaneEngineCore)
-![NuGet Downloads](https://img.shields.io/nuget/dt/OctaneEngineCore)
-
-![alt tag](https://image.ibb.co/h2tK8v/Untitled_1.png)
+![Nuget](https://img.shields.io/nuget/dt/OctaneEngineCore)
 
 
 A high Performance C# file downloader that asyncrounously downloads files as pieces. Made as a faster, more efficent replacement to Microsoft's WebClient.Want to see the library in action? Check out [Octane YouTube Extractor](https://github.com/gregyjames/OCTANE-YoutubeExtractor)
@@ -27,47 +24,49 @@ dotnet add package OctaneEngineCore
 
 # Usage
 ```csharp
-private const string Url = "https://plugins.jetbrains.com/files/7973/281233/sonarlint-intellij-7.4.0.60471.zip?updateId=281233&pluginId=7973&family=INTELLIJ";
-
-private static void Main(){
-     #region Logging Configuration
-          var seriLog = new LoggerConfiguration()
-               .Enrich.FromLogContext()
-               .MinimumLevel.Error()
-               .WriteTo.Async(a => a.File("./OctaneLog.txt"))
-               .WriteTo.Async(a => a.Console(theme: AnsiConsoleTheme.Sixteen))
-               .CreateLogger();
-            var factory = LoggerFactory.Create(logging =>
-            {
-                logging.AddSerilog(seriLog);
-            });
-     #endregion
-
-     #region Configuration Loading
-          var builder = new ConfigurationBuilder();
-          builder.SetBasePath(Directory.GetCurrentDirectory())
-               .AddJsonFile("appsettings.json", true, true);
-          var configRoot = builder.Build();
-     #endregion
-
-     #region Find and Set optimal number of parts
-          //var optimalNumberOfParts = Engine.GetOptimalNumberOfParts(Url).Result;
-          //seriLog.Information("Optimal number of parts to download file: {OptimalNumberOfParts}", optimalNumberOfParts);
-          //seriLog.Information("Speed: {Result}", NetworkAnalyzer.GetCurrentNetworkSpeed().Result);
-          //seriLog.Information("Latency: {Result}", NetworkAnalyzer.GetCurrentNetworkLatency().Result);
-     #endregion
-
-     var pauseTokenSource = new PauseTokenSource();
-     using var cancelTokenSource = new CancellationTokenSource();
-     var containerBuilder = new ContainerBuilder();
-     containerBuilder.RegisterInstance(factory).As<ILoggerFactory>();
-     containerBuilder.RegisterInstance(configRoot).As<IConfiguration>();
-     containerBuilder.AddOctane();
-     var engineContainer = containerBuilder.Build();
-     var engine = engineContainer.Resolve<IEngine>();
-     engine.DownloadFile(Url, null, pauseTokenSource, cancelTokenSource).Wait();
-}
-
+#region Logging Configuration
+var seriLog = new LoggerConfiguration()
+    .Enrich.FromLogContext()
+    .MinimumLevel.Error()
+    .WriteTo.Async(a => a.File("./OctaneLog.txt"))
+    .WriteTo.Async(a => a.Console(theme: AnsiConsoleTheme.Sixteen))
+    .CreateLogger();
+    var factory = LoggerFactory.Create(logging => {
+        logging.AddSerilog(seriLog);
+    });
+#endregion
+
+#region Configuration Loading
+    var builder = new ConfigurationBuilder();
+    builder.SetBasePath(Directory.GetCurrentDirectory())
+        .AddJsonFile("appsettings.json", true, true);
+    var configRoot = builder.Build();
+#endregion
+
+#region Find and Set optimal number of parts
+    //var optimalNumberOfParts = Engine.GetOptimalNumberOfParts(Url).Result;
+    //seriLog.Information("Optimal number of parts to download file: {OptimalNumberOfParts}", optimalNumberOfParts);
+#endregion
+
+//seriLog.Information("Speed: {Result}", NetworkAnalyzer.GetCurrentNetworkSpeed().Result);
+//seriLog.Information("Latency: {Result}", NetworkAnalyzer.GetCurrentNetworkLatency().Result);
+var pauseTokenSource = new PauseTokenSource();
+using var cancelTokenSource = new CancellationTokenSource();
+var containerBuilder = new ContainerBuilder();
+containerBuilder.RegisterInstance(factory).As<ILoggerFactory>();
+containerBuilder.RegisterInstance(configRoot).As<IConfiguration>();
+containerBuilder.AddOctane();
+var engineContainer = containerBuilder.Build();
+var engine = engineContainer.Resolve<IEngine>();
+engine.DownloadFile(
+    new OctaneRequest(){
+        URL = Url,
+        Headers = new Dictionary<string, string>(){}
+    }, 
+    null, 
+    pauseTokenSource, 
+    cancelTokenSource
+).Wait();
 ```
 
 # Benchmark
-Original file line number
+Diff line change
@@ Expand Up / @@ -30,7 +30,7 @@ protected override void Load(ContainerBuilder builder) @@
                 {
                     MaxResponseContentBufferSize = cfg.BufferSize,
                 };
                 return _client;
             }).As<HttpClient>().SingleInstance();
         }
@@ Expand Down @@