Merge pull request #105 from iPromKnight/community-powered-idea

Woke up to see a discussion and comments such as: powered by community tagline

.gitignore

@@ -405,3 +405,5 @@ FodyWeavers.xsd
 
 dist/
 deployment/docker/docker-compose-dev.yaml
+
+src/producer/.run/

src/producer/Configuration/scrapers.json

@@ -30,6 +30,11 @@
         "Name": "SyncDmmJob",
         "IntervalSeconds": 1800,
         "Enabled": true
+      },
+      {
+        "Name": "SyncTorrentioJob",
+        "IntervalSeconds": 604800,
+        "Enabled": true
       }
     ]
   }

src/producer/Configuration/torrentio.json

@@ -0,0 +1,14 @@
+{
+  "TorrentioConfiguration": {
+    "Instances": [
+      {
+        "Name": "Official",
+        "Url": "https://torrentio.strem.fun",
+        "RateLimit": {
+          "RequestLimit": 300,
+          "IntervalInSeconds": 3600
+        }
+      }
+    ]
+  }
+}

src/producer/Extensions/ConfigurationExtensions.cs

@@ -11,11 +11,36 @@ public static IConfigurationBuilder AddScrapeConfiguration(this IConfigurationBu
 
         configuration.AddJsonFile(LoggingConfig, false, true);
         configuration.AddJsonFile(ScrapeConfiguration.Filename, false, true);
+        configuration.AddJsonFile(TorrentioConfiguration.Filename, false, true);
 
         configuration.AddEnvironmentVariables();
 
         configuration.AddUserSecrets<Program>();
 
         return configuration;
     }
+
+    public static TConfiguration LoadConfigurationFromConfig<TConfiguration>(this IServiceCollection services, IConfiguration configuration, string sectionName)
+        where TConfiguration : class
+    {
+        var instance = configuration.GetSection(sectionName).Get<TConfiguration>();
+
+        ArgumentNullException.ThrowIfNull(instance, nameof(instance));
+
+        services.TryAddSingleton(instance);
+
+        return instance;
+    }
+
+    public static TConfiguration LoadConfigurationFromEnv<TConfiguration>(this IServiceCollection services)
+        where TConfiguration : class
+    {
+        var instance = Activator.CreateInstance<TConfiguration>();
+
+        ArgumentNullException.ThrowIfNull(instance, nameof(instance));
+
+        services.TryAddSingleton(instance);
+
+        return instance;
+    }
 }

src/producer/Interfaces/IMessagePublisher.cs → src/producer/Features/Amqp/IMessagePublisher.cs

@@ -1,4 +1,4 @@
-namespace Producer.Interfaces;
+namespace Producer.Features.Amqp;
 
 public interface IMessagePublisher
 {

src/producer/Jobs/PublisherJob.cs → src/producer/Features/Amqp/PublisherJob.cs

@@ -1,11 +1,14 @@
-namespace Producer.Jobs;
+using Literals = Producer.Features.JobSupport.Literals;
+
+namespace Producer.Features.Amqp;
 
 [DisallowConcurrentExecution]
+[ManualJobRegistration]
 public class PublisherJob(IMessagePublisher publisher, IDataStorage storage, ILogger<PublisherJob> logger) : IJob
 {
     private const string JobName = nameof(PublisherJob);
-    public static readonly JobKey Key = new(JobName, nameof(Jobs));
-    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Jobs));
+    public static readonly JobKey Key = new(JobName, nameof(Literals.PublishingJobs));
+    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Literals.PublishingJobs));
 
     public async Task Execute(IJobExecutionContext context)
     {

src/producer/Models/Configuration/RabbitMqConfiguration.cs → src/producer/Features/Amqp/RabbitMqConfiguration.cs

@@ -1,4 +1,4 @@
-namespace Producer.Models.Configuration;
+namespace Producer.Features.Amqp;
 
 public class RabbitMqConfiguration
 {

src/producer/Features/Amqp/ServiceCollectionExtensions.cs

@@ -0,0 +1,24 @@
+namespace Producer.Features.Amqp;
+
+internal static class ServiceCollectionExtensions
+{
+    internal static IServiceCollection RegisterMassTransit(this IServiceCollection services)
+    {
+        var rabbitConfig = services.LoadConfigurationFromEnv<RabbitMqConfiguration>();
+
+        services.AddMassTransit(busConfigurator =>
+        {
+            busConfigurator.SetKebabCaseEndpointNameFormatter();
+            busConfigurator.UsingRabbitMq((_, busFactoryConfigurator) =>
+            {
+                busFactoryConfigurator.Host(rabbitConfig.Host, hostConfigurator =>
+                {
+                    hostConfigurator.Username(rabbitConfig.Username);
+                    hostConfigurator.Password(rabbitConfig.Password);
+                });
+            });
+        });
+
+        return services;
+    }
+}

src/producer/Services/TorrentPublisher.cs → src/producer/Features/Amqp/TorrentPublisher.cs

@@ -1,4 +1,4 @@
-namespace Producer.Services;
+namespace Producer.Features.Amqp;
 
 public class TorrentPublisher(
     ISendEndpointProvider sendEndpointProvider,

src/producer/Crawlers/BaseCrawler.cs → src/producer/Features/CrawlerSupport/BaseCrawler.cs

@@ -1,4 +1,4 @@
-namespace Producer.Crawlers;
+namespace Producer.Features.CrawlerSupport;
 
 public abstract class BaseCrawler(ILogger<BaseCrawler> logger, IDataStorage storage) : ICrawler
 {

src/producer/Crawlers/BaseJsonCrawler.cs → src/producer/Features/CrawlerSupport/BaseJsonCrawler.cs

@@ -1,8 +1,8 @@
-namespace Producer.Crawlers;
+namespace Producer.Features.CrawlerSupport;
 
 public abstract class BaseJsonCrawler(IHttpClientFactory httpClientFactory, ILogger<BaseJsonCrawler> logger, IDataStorage storage) : BaseCrawler(logger, storage)
 {
-    private readonly HttpClient _client = httpClientFactory.CreateClient("Scraper");
+    private readonly HttpClient _client = httpClientFactory.CreateClient(Literals.CrawlerClient);
 
     protected virtual async Task Execute(string collectionName)
     {

src/producer/Crawlers/BaseXmlCrawler.cs → src/producer/Features/CrawlerSupport/BaseXmlCrawler.cs

@@ -1,12 +1,12 @@
-namespace Producer.Crawlers;
+namespace Producer.Features.CrawlerSupport;
 
 public abstract class BaseXmlCrawler(IHttpClientFactory httpClientFactory, ILogger<BaseXmlCrawler> logger, IDataStorage storage) : BaseCrawler(logger, storage)
 {
     public override async Task Execute()
     {
         logger.LogInformation("Starting {Source} crawl", Source);
 
-        using var client = httpClientFactory.CreateClient("Scraper");
+        using var client = httpClientFactory.CreateClient(Literals.CrawlerClient);
         var xml = await client.GetStringAsync(Url);
         var xmlRoot = XElement.Parse(xml);
 

src/producer/Crawlers/CrawlerProvider.cs → src/producer/Features/CrawlerSupport/CrawlerProvider.cs

@@ -1,4 +1,4 @@
-namespace Producer.Crawlers;
+namespace Producer.Features.CrawlerSupport;
 
 public class CrawlerProvider(IServiceProvider serviceProvider) : ICrawlerProvider
 {

src/producer/Interfaces/ICrawler.cs → src/producer/Features/CrawlerSupport/ICrawler.cs

@@ -1,4 +1,4 @@
-namespace Producer.Interfaces;
+namespace Producer.Features.CrawlerSupport;
 
 public interface ICrawler
 {

src/producer/Interfaces/ICrawlerProvider.cs → src/producer/Features/CrawlerSupport/ICrawlerProvider.cs

@@ -1,4 +1,4 @@
-namespace Producer.Interfaces;
+namespace Producer.Features.CrawlerSupport;
 
 public interface ICrawlerProvider
 {

src/producer/Features/CrawlerSupport/Literals.cs

@@ -0,0 +1,6 @@
+namespace Producer.Features.CrawlerSupport;
+
+public static class Literals
+{
+    public const string CrawlerClient = "Scraper";
+}

src/producer/Models/Results.cs → src/producer/Features/CrawlerSupport/Results.cs

@@ -1,4 +1,4 @@
-namespace Producer.Models;
+namespace Producer.Features.CrawlerSupport;
 
 public record InsertTorrentResult(bool Success, int InsertedCount = 0, string? ErrorMessage = null);
 public record UpdatedTorrentResult(bool Success, int UpdatedCount = 0, string? ErrorMessage = null);

src/producer/Models/Scraper.cs → src/producer/Features/CrawlerSupport/Scraper.cs

@@ -1,4 +1,4 @@
-namespace Producer.Models;
+namespace Producer.Features.CrawlerSupport;
 
 public class Scraper
 {

src/producer/Features/CrawlerSupport/ServiceCollectionExtensions.cs

@@ -0,0 +1,23 @@
+namespace Producer.Features.CrawlerSupport;
+
+internal static class ServiceCollectionExtensions
+{
+    internal static IServiceCollection AddCrawlers(this IServiceCollection services)
+    {
+        services.AddHttpClient(Literals.CrawlerClient);
+
+        var crawlerTypes = Assembly.GetAssembly(typeof(ICrawler))
+            .GetTypes()
+            .Where(t => t is {IsClass: true, IsAbstract: false} && typeof(ICrawler).IsAssignableFrom(t));
+
+        foreach (var type in crawlerTypes)
+        {
+            services.AddKeyedTransient(typeof(ICrawler), type.Name, type);
+        }
+
+        services
+            .AddSingleton<ICrawlerProvider, CrawlerProvider>();
+
+        return services;
+    }
+}

src/producer/Models/Torrent.cs → src/producer/Features/CrawlerSupport/Torrent.cs

@@ -1,4 +1,4 @@
-namespace Producer.Models;
+namespace Producer.Features.CrawlerSupport;
 
 // Torrent represents a crawled torrent from one of our
 // supported sources.