diff --git a/src/debrid-collector/Dockerfile b/src/debrid-collector/Dockerfile index 768a90cb..d695785a 100644 --- a/src/debrid-collector/Dockerfile +++ b/src/debrid-collector/Dockerfile @@ -15,7 +15,7 @@ WORKDIR /app ENV PYTHONUNBUFFERED=1 -RUN apk add --update --no-cache python3=~3.11.9-r0 py3-pip && ln -sf python3 /usr/bin/python +RUN apk add --update --no-cache python3=~3.11 py3-pip && ln -sf python3 /usr/bin/python COPY --from=build /src/out . diff --git a/src/metadata/Features/Configuration/PostgresConfiguration.cs b/src/metadata/Features/Configuration/PostgresConfiguration.cs index c7e465c2..826a9d97 100644 --- a/src/metadata/Features/Configuration/PostgresConfiguration.cs +++ b/src/metadata/Features/Configuration/PostgresConfiguration.cs @@ -8,12 +8,14 @@ public class PostgresConfiguration private const string PasswordVariable = "PASSWORD"; private const string DatabaseVariable = "DB"; private const string PortVariable = "PORT"; + private const string CommandTimeoutVariable = "COMMAND_TIMEOUT_SEC"; // Seconds private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable); private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable); private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable); private string Database { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DatabaseVariable); private int PORT { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 5432); + private int CommandTimeout { get; init; } = Prefix.GetEnvironmentVariableAsInt(CommandTimeoutVariable, 300); - public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};"; + public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};CommandTimeout={CommandTimeout}"; } diff --git a/src/metadata/Features/ImportImdbData/ImdbDbService.cs b/src/metadata/Features/ImportImdbData/ImdbDbService.cs index 4c584c0c..ce01782f 100644 --- a/src/metadata/Features/ImportImdbData/ImdbDbService.cs +++ b/src/metadata/Features/ImportImdbData/ImdbDbService.cs @@ -134,7 +134,7 @@ private async Task ExecuteCommandAsync(Func operation, s { try { - await using var connection = CreateNpgsqlConnection(); + await using var connection = new NpgsqlConnection(configuration.StorageConnectionString); await connection.OpenAsync(); await operation(connection); @@ -145,16 +145,6 @@ private async Task ExecuteCommandAsync(Func operation, s } } - private NpgsqlConnection CreateNpgsqlConnection() - { - var connectionStringBuilder = new NpgsqlConnectionStringBuilder(configuration.StorageConnectionString) - { - CommandTimeout = 3000, - }; - - return new(connectionStringBuilder.ConnectionString); - } - private async Task ExecuteCommandWithTransactionAsync(Func operation, NpgsqlTransaction transaction, string errorMessage) { try diff --git a/src/metadata/Metadata.csproj b/src/metadata/Metadata.csproj index 7f17029e..c3703302 100644 --- a/src/metadata/Metadata.csproj +++ b/src/metadata/Metadata.csproj @@ -13,7 +13,7 @@ - + diff --git a/src/producer/src/Dockerfile b/src/producer/src/Dockerfile index 8cd57301..3b5ef0ca 100644 --- a/src/producer/src/Dockerfile +++ b/src/producer/src/Dockerfile @@ -14,7 +14,7 @@ WORKDIR /app ENV PYTHONUNBUFFERED=1 -RUN apk add --update --no-cache python3=~3.11.9-r0 py3-pip && ln -sf python3 /usr/bin/python +RUN apk add --update --no-cache python3=~3.11 py3-pip && ln -sf python3 /usr/bin/python COPY --from=build /src/out . diff --git a/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs b/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs index c690c9ff..393950e0 100644 --- a/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs +++ b/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs @@ -14,7 +14,7 @@ public partial class DebridMediaManagerCrawler( protected override string Source => "DMM"; private const int ParallelismCount = 4; - + public override async Task Execute() { var tempDirectory = await dmmFileDownloader.DownloadFileToTempPath(CancellationToken.None); @@ -24,7 +24,7 @@ public override async Task Execute() logger.LogInformation("Found {Files} files to parse", files.Length); var options = new ParallelOptions { MaxDegreeOfParallelism = ParallelismCount }; - + await Parallel.ForEachAsync(files, options, async (file, token) => { var fileName = Path.GetFileName(file); @@ -69,9 +69,9 @@ await Parallel.ForEachAsync(batchProcessables.Select(t => t.InfoHash), options, if (page.TryGetValue(infoHash, out var dmmContent) && successfulResponses.TryGetValue(dmmContent.Filename, out var parsedResponse)) { - page[infoHash] = dmmContent with {ParseResponse = parsedResponse}; + page[infoHash] = dmmContent with { ParseResponse = parsedResponse }; } - + return ValueTask.CompletedTask; }); } @@ -86,7 +86,7 @@ await Parallel.ForEachAsync(batchProcessables.Select(t => t.InfoHash), options, } var pageSource = await File.ReadAllTextAsync(filePath); - + var match = HashCollectionMatcher().Match(pageSource); if (!match.Success) @@ -106,9 +106,34 @@ await Parallel.ForEachAsync(batchProcessables.Select(t => t.InfoHash), options, var decodedJson = LZString.DecompressFromEncodedURIComponent(encodedJson.Value); - var json = JsonDocument.Parse(decodedJson); - - var torrents = await json.RootElement.EnumerateArray() + JsonElement arrayToProcess; + try + { + var json = JsonDocument.Parse(decodedJson); + + if (json.RootElement.ValueKind == JsonValueKind.Object && + json.RootElement.TryGetProperty("torrents", out var torrentsProperty) && + torrentsProperty.ValueKind == JsonValueKind.Array) + { + arrayToProcess = torrentsProperty; + } + else if (json.RootElement.ValueKind == JsonValueKind.Array) + { + arrayToProcess = json.RootElement; + } + else + { + logger.LogWarning("Unexpected JSON format in {Name}", name); + return []; + } + } + catch (Exception ex) + { + logger.LogError("Failed to parse JSON {decodedJson} for {Name}: {Exception}", decodedJson, name, ex); + return []; + } + + var torrents = await arrayToProcess.EnumerateArray() .ToAsyncEnumerable() .Select(ParsePageContent) .Where(t => t is not null) @@ -120,7 +145,7 @@ await Parallel.ForEachAsync(batchProcessables.Select(t => t.InfoHash), options, await Storage.MarkPageAsIngested(filenameOnly); return []; } - + var torrentDictionary = torrents .Where(x => x is not null) .GroupBy(x => x.InfoHash) @@ -141,7 +166,7 @@ await Parallel.ForEachAsync(page, options, async (kvp, ct) => { var (infoHash, dmmContent) = kvp; var parsedTorrent = dmmContent.ParseResponse; - if (parsedTorrent is not {Success: true}) + if (parsedTorrent is not { Success: true }) { return; } @@ -192,7 +217,7 @@ private IngestedTorrent MapToTorrent(ImdbEntry result, long size, string infoHas Category = AssignCategory(result), RtnResponse = parsedTorrent.Response.ToJson(), }; - + private Task AddToCache(string cacheKey, ImdbEntry best) { @@ -200,19 +225,19 @@ private Task AddToCache(string cacheKey, ImdbEntry best) { AbsoluteExpirationRelativeToNow = TimeSpan.FromDays(1), }; - + return cache.SetStringAsync(cacheKey, JsonSerializer.Serialize(best), cacheOptions); } private async Task<(bool Success, ImdbEntry? Entry)> CheckIfInCacheAndReturn(string cacheKey) { var cachedImdbId = await cache.GetStringAsync(cacheKey); - + if (!string.IsNullOrEmpty(cachedImdbId)) { return (true, JsonSerializer.Deserialize(cachedImdbId)); } - + return (false, null); } @@ -222,7 +247,7 @@ private Task AddToCache(string cacheKey, ImdbEntry best) return (pageIngested, filename); } - + private static string AssignCategory(ImdbEntry entry) => entry.Category.ToLower() switch { @@ -230,9 +255,9 @@ var category when string.Equals(category, "movie", StringComparison.OrdinalIgnor var category when string.Equals(category, "tvSeries", StringComparison.OrdinalIgnoreCase) => "tv", _ => "unknown", }; - + private static string GetCacheKey(string category, string title, int year) => $"{category.ToLowerInvariant()}:{year}:{title.ToLowerInvariant()}"; - + private static ExtractedDMMContent? ParsePageContent(JsonElement item) { if (!item.TryGetProperty("filename", out var filenameElement) || @@ -241,10 +266,10 @@ var category when string.Equals(category, "tvSeries", StringComparison.OrdinalIg { return null; } - + return new(filenameElement.GetString(), bytesElement.GetInt64(), hashElement.GetString()); } - + private record DmmContent(string Filename, long Bytes, ParseTorrentTitleResponse? ParseResponse); private record ExtractedDMMContent(string Filename, long Bytes, string InfoHash); private record RtnBatchProcessable(string InfoHash, string Filename); diff --git a/src/qbit-collector/Dockerfile b/src/qbit-collector/Dockerfile index e5d6fca7..253af062 100644 --- a/src/qbit-collector/Dockerfile +++ b/src/qbit-collector/Dockerfile @@ -15,7 +15,7 @@ WORKDIR /app ENV PYTHONUNBUFFERED=1 -RUN apk add --update --no-cache python3=~3.11.9-r0 py3-pip && ln -sf python3 /usr/bin/python +RUN apk add --update --no-cache python3=~3.11 py3-pip && ln -sf python3 /usr/bin/python COPY --from=build /src/out . diff --git a/src/shared/Configuration/PostgresConfiguration.cs b/src/shared/Configuration/PostgresConfiguration.cs index ca12d592..069248db 100644 --- a/src/shared/Configuration/PostgresConfiguration.cs +++ b/src/shared/Configuration/PostgresConfiguration.cs @@ -8,12 +8,14 @@ public class PostgresConfiguration private const string PasswordVariable = "PASSWORD"; private const string DatabaseVariable = "DB"; private const string PortVariable = "PORT"; + private const string CommandTimeoutVariable = "COMMAND_TIMEOUT_SEC"; // Seconds private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable); private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable); private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable); private string Database { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DatabaseVariable); private int PORT { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 5432); + private int CommandTimeout { get; init; } = Prefix.GetEnvironmentVariableAsInt(CommandTimeoutVariable, 300); - public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};"; + public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};CommandTimeout={CommandTimeout}"; } diff --git a/src/shared/SharedContracts.csproj b/src/shared/SharedContracts.csproj index ed1ef643..65cbf921 100644 --- a/src/shared/SharedContracts.csproj +++ b/src/shared/SharedContracts.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/tissue/Features/DataProcessing/PostgresConfiguration.cs b/src/tissue/Features/DataProcessing/PostgresConfiguration.cs index a878c0e2..b37e3d1d 100644 --- a/src/tissue/Features/DataProcessing/PostgresConfiguration.cs +++ b/src/tissue/Features/DataProcessing/PostgresConfiguration.cs @@ -8,12 +8,14 @@ public class PostgresConfiguration private const string PasswordVariable = "PASSWORD"; private const string DatabaseVariable = "DB"; private const string PortVariable = "PORT"; + private const string CommandTimeoutVariable = "COMMAND_TIMEOUT_SEC"; // Seconds private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable); private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable); private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable); private string Database { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DatabaseVariable); private int PORT { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 5432); + private int CommandTimeout { get; init; } = Prefix.GetEnvironmentVariableAsInt(CommandTimeoutVariable, 300); - public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};"; + public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};CommandTimeout={CommandTimeout}"; } diff --git a/src/tissue/Tissue.csproj b/src/tissue/Tissue.csproj index cb658650..664e36c0 100644 --- a/src/tissue/Tissue.csproj +++ b/src/tissue/Tissue.csproj @@ -12,7 +12,7 @@ - +