feat(MangaDex): add retry logic for paged requests

backend/MangaMagnet.Api/MangaMagnet.Api.csproj

@@ -25,6 +25,7 @@
           <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
         <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.0" />
+        <PackageReference Include="Polly.Extensions" Version="8.3.0" />
         <PackageReference Include="Quartz.AspNetCore" Version="3.8.0" />
         <PackageReference Include="Quartz.Serialization.Json" Version="3.8.0" />
         <PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />

backend/MangaMagnet.Api/Program.cs

@@ -18,6 +18,8 @@
 using MangaMagnet.Core.Services;
 using Microsoft.EntityFrameworkCore;
 using Npgsql;
+using Polly;
+using Polly.Retry;
 using Quartz;
 using Quartz.Impl.AdoJobStore;
 using Serilog;
@@ -59,6 +61,13 @@
 builder.Services.AddScoped<MetadataService>();
 builder.Services.AddScoped<DownloadService>();
 
+// Polly
+builder.Services.AddResiliencePipeline("MangaDex-Pipeline", pipelineBuilder =>
+{
+	pipelineBuilder
+		.AddRetry(new RetryStrategyOptions{ MaxRetryAttempts = 3});
+});
+
 
 // Progress services
 builder.Services.AddSingleton<ProgressService>();

backend/MangaMagnet.Api/Service/MetadataService.cs

@@ -67,6 +67,8 @@ public async Task<IEnumerable<MangaMetadataResponse>> UpdateAllMetadataAsync(Can
 			progressTask.Increment();
 		}
 
+		logger.LogDebug("Updated all metadata");
+
 		return updated;
 	}
 

backend/MangaMagnet.Api/appsettings.Development.json

@@ -7,7 +7,8 @@
 				"Microsoft.EntityFrameworkCore.Database": "Warning",
 				"Microsoft.EntityFrameworkCore.Infrastructure": "Warning",
 				"System.Net.Http.HttpClient": "Warning",
-				"MangaMagnet": "Debug"
+				"MangaMagnet": "Debug",
+				"Polly": "Error"
 			}
 		}
 	},

backend/MangaMagnet.Api/packages.lock.json

@@ -81,6 +81,18 @@
           "Npgsql": "8.0.0"
         }
       },
+      "Polly.Extensions": {
+        "type": "Direct",
+        "requested": "[8.3.0, )",
+        "resolved": "8.3.0",
+        "contentHash": "wUjK/vOqq2kaj9WTtBc/tXWykgT3Z5qpvM8ozl6UEBrIsjbatCl6a+AXMZsTwVEyqUKFF9td3aYRmSAaBDtRrQ==",
+        "dependencies": {
+          "Microsoft.Extensions.Logging.Abstractions": "8.0.0",
+          "Microsoft.Extensions.Options": "8.0.0",
+          "Polly.Core": "8.3.0",
+          "System.Diagnostics.DiagnosticSource": "8.0.0"
+        }
+      },
       "Quartz.AspNetCore": {
         "type": "Direct",
         "requested": "[3.8.0, )",
@@ -678,6 +690,19 @@
         "resolved": "1.0.12",
         "contentHash": "f1nye8fMJCYfJeR+Qb7qlcvC8et4Y4TLf2GLfaFoeISgI+yisaJkbNSGfKCpFBNeVF/iPDkyFmr4J28dv342/Q=="
       },
+      "Polly": {
+        "type": "Transitive",
+        "resolved": "8.3.0",
+        "contentHash": "vqAQmLO1gZcS8Ox9Ky+hrq0HxYrYVSBS1vzv6o6RZH3vvECutngIxkUHw9YijbG0QrycxvHo0Hbx0/D7MGgdow==",
+        "dependencies": {
+          "Polly.Core": "8.3.0"
+        }
+      },
+      "Polly.Core": {
+        "type": "Transitive",
+        "resolved": "8.3.0",
+        "contentHash": "A3r4xkWasffyrIJGjA3jWQi86ZGcR9BEbpIIoP9xYRAKYwiaGbKpqpsHdWv5sCeLZdaWpVs7pc4EM8Tei+LluA=="
+      },
       "Quartz": {
         "type": "Transitive",
         "resolved": "3.8.0",
@@ -1163,6 +1188,7 @@
           "ExtendedXmlSerializer": "[3.7.11, )",
           "Microsoft.EntityFrameworkCore": "[8.0.1, )",
           "Microsoft.Extensions.Http": "[8.0.0, )",
+          "Polly": "[8.3.0, )",
           "System.Threading.RateLimiting": "[8.0.0, )"
         }
       }

backend/MangaMagnet.Core/MangaMagnet.Core.csproj

@@ -14,6 +14,7 @@
       <PackageReference Include="ExtendedXmlSerializer" Version="3.7.11" />
       <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.1" />
       <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
+      <PackageReference Include="Polly" Version="8.3.0" />
       <PackageReference Include="System.Threading.RateLimiting" Version="8.0.0" />
     </ItemGroup>
 

backend/MangaMagnet.Core/Providers/MangaDex/MangaDexApiService.cs

@@ -14,10 +14,13 @@
 using MangaMagnet.Core.Providers.MangaDex.Models.Download;
 using MangaMagnet.Core.Util;
 using Microsoft.Extensions.Logging;
+using Polly;
+using Polly.Registry;
+using Polly.Retry;
 
 namespace MangaMagnet.Core.Providers.MangaDex;
 
-public class MangaDexApiService(IHttpClientFactory httpClientFactory, ILogger<MangaDexApiService> logger)
+public class MangaDexApiService(IHttpClientFactory httpClientFactory, ILogger<MangaDexApiService> logger, ResiliencePipelineProvider<string> resiliencePipelineProvider)
 	: SimpleRatelimitedProvider
 {
 	public async Task<List<MangaDexMangaData>> SearchMangaByNameAsync(string input,
@@ -139,14 +142,29 @@ private async Task<List<T>> FetchAllPagesAsync<T>(string mangaDexId, Func<string
 		{
 			const int limit = 100;
 
-			var response = await func(mangaDexId, offset, limit, cancellationToken);
+			var pipeline = resiliencePipelineProvider.GetPipeline("MangaDex-Pipeline");
 
-			var responseDataCount = response.Data.Count;
+			await pipeline.ExecuteAsync( async cancelToken =>
+			{
+				try
+				{
+					var response = await func(mangaDexId, offset, limit, cancelToken);
 
-			inner.AddRange(response.Data);
+					var responseDataCount = response.Data.Count;
 
-			offset += responseDataCount;
-			doNextRequest = response.Total > offset;
+					inner.AddRange(response.Data);
+
+					offset += responseDataCount;
+					doNextRequest = response.Total > offset;
+				}
+				catch (Exception e)
+				{
+					var page = offset / limit;
+
+					logger.LogWarning("Failed to fetch Page {Page}: {Exception}", page, e.Message);
+					throw;
+				}
+			}, cancellationToken);
 		}
 
 		return inner;