From 51a8b513763175beb3892a2714f4e9de21e79e5f Mon Sep 17 00:00:00 2001 From: xiaoxiao921 Date: Wed, 12 Oct 2022 13:13:31 +0200 Subject: [PATCH] Adds a method for trimming duplicate exceptions from logs --- CHEF/Components/Watcher/AutoPastebin.cs | 110 +++++++++++++----------- CHEF/Components/Watcher/CommonIssues.cs | 41 ++++++++- 2 files changed, 102 insertions(+), 49 deletions(-) diff --git a/CHEF/Components/Watcher/AutoPastebin.cs b/CHEF/Components/Watcher/AutoPastebin.cs index dc6e95b..66ecffd 100644 --- a/CHEF/Components/Watcher/AutoPastebin.cs +++ b/CHEF/Components/Watcher/AutoPastebin.cs @@ -6,7 +6,6 @@ using System.Net; using System.Net.Http; using System.Text; -using System.Threading; using System.Threading.Tasks; using CHEF.Components.Commands.Ignore; using Discord; @@ -60,71 +59,86 @@ static bool IsAttachmentValidExtension(string extension) return IsPasteBinValidExtension(extension) || extension == ".zip"; } - const int TwentyMiB = 20971520; - const int attachmentMaxFileSizeInBytes = TwentyMiB; - if (IsAttachmentValidExtension(fileType) && attachment.Size <= attachmentMaxFileSizeInBytes) + const int FiftyMiB = 52428800; + const int attachmentMaxFileSizeInBytes = FiftyMiB; + if (IsAttachmentValidExtension(fileType)) { - var fileContentStream = await _httpClient.GetStreamAsync(attachment.Url); - var botAnswer = new StringBuilder(); - - List fileContents = new(); - if (fileType == ".zip") + if (attachment.Size <= attachmentMaxFileSizeInBytes) { - Logger.Log("Scanning zip attachment"); - using (var zipArchive = new ZipArchive(fileContentStream)) + var fileContentStream = await _httpClient.GetStreamAsync(attachment.Url); + var botAnswer = new StringBuilder(); + + List fileContents = new(); + if (fileType == ".zip") { - foreach (var zipArchiveEntry in zipArchive.Entries) + Logger.Log("Scanning zip attachment"); + using (var zipArchive = new ZipArchive(fileContentStream)) { - Logger.Log($"Scanning zip entry: {zipArchiveEntry.FullName}"); - - var extension = Path.GetExtension(zipArchiveEntry.Name); - if (IsPasteBinValidExtension(extension)) + foreach (var zipArchiveEntry in zipArchive.Entries) { - if (zipArchiveEntry.Length > attachmentMaxFileSizeInBytes) + Logger.Log($"Scanning zip entry: {zipArchiveEntry.FullName}"); + + var extension = Path.GetExtension(zipArchiveEntry.Name); + if (IsPasteBinValidExtension(extension)) { - Logger.Log($"Skipping log scanning for file {zipArchiveEntry.Name} because file size is {zipArchiveEntry.Length}, max is {attachmentMaxFileSizeInBytes}"); - continue; + if (zipArchiveEntry.Length > attachmentMaxFileSizeInBytes) + { + Logger.Log($"Skipping log scanning for file {zipArchiveEntry.Name} because file size is {zipArchiveEntry.Length}, max is {attachmentMaxFileSizeInBytes}"); + continue; + } + + Logger.Log("Reading zip entry into a string"); + using var entryStream = zipArchiveEntry.Open(); + using var streamReader = new StreamReader(entryStream); + fileContents.Add(streamReader.ReadToEnd()); + break; } - - Logger.Log("Reading zip entry into a string"); - using var entryStream = zipArchiveEntry.Open(); - using var streamReader = new StreamReader(entryStream); - fileContents.Add(streamReader.ReadToEnd()); - break; } } } - } - else - { - Logger.Log($"Putting file attachment content into a string"); - using var streamReader = new StreamReader(fileContentStream); - fileContents.Add(streamReader.ReadToEnd()); - } + else + { + Logger.Log($"Putting file attachment content into a string"); + using var streamReader = new StreamReader(fileContentStream); + fileContents.Add(streamReader.ReadToEnd()); + } - foreach (var fileContent in fileContents) - { - if (!string.IsNullOrWhiteSpace(fileContent)) + foreach (var fileContent in fileContents) { - using (var context = new IgnoreContext()) + if (!string.IsNullOrWhiteSpace(fileContent)) { - if (!await context.IsIgnored(msg.Author)) - { - Logger.Log("Scanning log content for common issues"); - CommonIssues.CheckCommonLogError(fileContent, botAnswer, msg.Author); - await CommonIssues.CheckForOutdatedAndDeprecatedMods(fileContent, botAnswer, msg.Author); - } - else + var alreadyPostedBin = false; + + using (var context = new IgnoreContext()) { - Logger.Log($"Message author {msg.Author} is ignored, not scanning attachment"); + if (!await context.IsIgnored(msg.Author)) + { + Logger.Log("Scanning log content for common issues"); + CommonIssues.CheckCommonLogError(fileContent, botAnswer, msg.Author); + await CommonIssues.CheckForOutdatedAndDeprecatedMods(fileContent, botAnswer, msg.Author); + + var noDuplicateFileContent = CommonIssues.RemoveDuplicateExceptionsFromText(fileContent); + + _ = Task.Run(() => PostBin(msg, attachment, noDuplicateFileContent)); + alreadyPostedBin = true; + } + else + { + Logger.Log($"Message author {msg.Author} is ignored, not scanning attachment"); + } } - } - _ = Task.Run(() => PostBin(msg, attachment, fileContent)); + if (!alreadyPostedBin) + _ = Task.Run(() => PostBin(msg, attachment, fileContent)); + } } - } - return botAnswer.ToString(); + return botAnswer.ToString(); + } + else + { + return "That file is quite large, if you want it to be scanned by CHEF for common log errors, please relaunch the game and exit as soon as you hit the main menu screen, it will prevent the file from getting a lot bigger than needed."; + } } } diff --git a/CHEF/Components/Watcher/CommonIssues.cs b/CHEF/Components/Watcher/CommonIssues.cs index df7e415..c2ba386 100644 --- a/CHEF/Components/Watcher/CommonIssues.cs +++ b/CHEF/Components/Watcher/CommonIssues.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; @@ -65,7 +66,7 @@ public static async Task CheckForOutdatedAndDeprecatedMods(string text, St { var textContainsAnyBadMod = false; - if (text != null) + if (!string.IsNullOrWhiteSpace(text)) { const string ThunderstoreManifestPrefix = "TS Manifest: "; if (text.Contains(ThunderstoreManifestPrefix, StringComparison.InvariantCultureIgnoreCase)) @@ -134,6 +135,44 @@ public static async Task CheckForOutdatedAndDeprecatedMods(string text, St return textContainsAnyBadMod; } + public static string RemoveDuplicateExceptionsFromText(string text) + { + if (!string.IsNullOrWhiteSpace(text)) + { + using (var stringReader = new StringReader(text)) + { + var isLinePartOfAnException = false; + + HashSet exceptions = new(); + StringBuilder linesSb = new(); + + string line; + while ((line = stringReader.ReadLine()) != null) + { + if (line.Contains("Exception:", StringComparison.InvariantCulture)) + { + isLinePartOfAnException = true; + } + else if (line.StartsWith('[') && + line.Contains(':', StringComparison.InvariantCulture) && + line.Contains(']', StringComparison.InvariantCulture)) + { + isLinePartOfAnException = false; + } + + if (!isLinePartOfAnException || exceptions.Add(line)) + { + linesSb.AppendLine(line); + } + } + + return linesSb.ToString(); + } + } + + return null; + } + /// /// Check for the if is the latest version. /// Returns the latest version as a string, null if is the latest.