Skip to content

Commit

Permalink
Adds a method for trimming duplicate exceptions from logs
Browse files Browse the repository at this point in the history
  • Loading branch information
@xiaoxiao921
xiaoxiao921 committed Oct 12, 2022
1 parent f62fdb4 commit 51a8b51
Show file tree
Hide file tree
Showing 2 changed files with 102 additions and 49 deletions.
110 changes: 62 additions & 48 deletions CHEF/Components/Watcher/AutoPastebin.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using CHEF.Components.Commands.Ignore;
using Discord;
Expand Down Expand Up @@ -60,71 +59,86 @@ static bool IsAttachmentValidExtension(string extension)
return IsPasteBinValidExtension(extension) || extension == ".zip";
}

const int TwentyMiB = 20971520;
const int attachmentMaxFileSizeInBytes = TwentyMiB;
if (IsAttachmentValidExtension(fileType) && attachment.Size <= attachmentMaxFileSizeInBytes)
const int FiftyMiB = 52428800;
const int attachmentMaxFileSizeInBytes = FiftyMiB;
if (IsAttachmentValidExtension(fileType))
{
var fileContentStream = await _httpClient.GetStreamAsync(attachment.Url);
var botAnswer = new StringBuilder();

List<string> fileContents = new();
if (fileType == ".zip")
if (attachment.Size <= attachmentMaxFileSizeInBytes)
{
Logger.Log("Scanning zip attachment");
using (var zipArchive = new ZipArchive(fileContentStream))
var fileContentStream = await _httpClient.GetStreamAsync(attachment.Url);
var botAnswer = new StringBuilder();

List<string> fileContents = new();
if (fileType == ".zip")
{
foreach (var zipArchiveEntry in zipArchive.Entries)
Logger.Log("Scanning zip attachment");
using (var zipArchive = new ZipArchive(fileContentStream))
{
Logger.Log($"Scanning zip entry: {zipArchiveEntry.FullName}");

var extension = Path.GetExtension(zipArchiveEntry.Name);
if (IsPasteBinValidExtension(extension))
foreach (var zipArchiveEntry in zipArchive.Entries)
{
if (zipArchiveEntry.Length > attachmentMaxFileSizeInBytes)
Logger.Log($"Scanning zip entry: {zipArchiveEntry.FullName}");

var extension = Path.GetExtension(zipArchiveEntry.Name);
if (IsPasteBinValidExtension(extension))
{
Logger.Log($"Skipping log scanning for file {zipArchiveEntry.Name} because file size is {zipArchiveEntry.Length}, max is {attachmentMaxFileSizeInBytes}");
continue;
if (zipArchiveEntry.Length > attachmentMaxFileSizeInBytes)
{
Logger.Log($"Skipping log scanning for file {zipArchiveEntry.Name} because file size is {zipArchiveEntry.Length}, max is {attachmentMaxFileSizeInBytes}");
continue;
}

Logger.Log("Reading zip entry into a string");
using var entryStream = zipArchiveEntry.Open();
using var streamReader = new StreamReader(entryStream);
fileContents.Add(streamReader.ReadToEnd());
break;
}

Logger.Log("Reading zip entry into a string");
using var entryStream = zipArchiveEntry.Open();
using var streamReader = new StreamReader(entryStream);
fileContents.Add(streamReader.ReadToEnd());
break;
}
}
}
}
else
{
Logger.Log($"Putting file attachment content into a string");
using var streamReader = new StreamReader(fileContentStream);
fileContents.Add(streamReader.ReadToEnd());
}
else
{
Logger.Log($"Putting file attachment content into a string");
using var streamReader = new StreamReader(fileContentStream);
fileContents.Add(streamReader.ReadToEnd());
}

foreach (var fileContent in fileContents)
{
if (!string.IsNullOrWhiteSpace(fileContent))
foreach (var fileContent in fileContents)
{
using (var context = new IgnoreContext())
if (!string.IsNullOrWhiteSpace(fileContent))
{
if (!await context.IsIgnored(msg.Author))
{
Logger.Log("Scanning log content for common issues");
CommonIssues.CheckCommonLogError(fileContent, botAnswer, msg.Author);
await CommonIssues.CheckForOutdatedAndDeprecatedMods(fileContent, botAnswer, msg.Author);
}
else
var alreadyPostedBin = false;

using (var context = new IgnoreContext())
{
Logger.Log($"Message author {msg.Author} is ignored, not scanning attachment");
if (!await context.IsIgnored(msg.Author))
{
Logger.Log("Scanning log content for common issues");
CommonIssues.CheckCommonLogError(fileContent, botAnswer, msg.Author);
await CommonIssues.CheckForOutdatedAndDeprecatedMods(fileContent, botAnswer, msg.Author);

var noDuplicateFileContent = CommonIssues.RemoveDuplicateExceptionsFromText(fileContent);

_ = Task.Run(() => PostBin(msg, attachment, noDuplicateFileContent));
alreadyPostedBin = true;
}
else
{
Logger.Log($"Message author {msg.Author} is ignored, not scanning attachment");
}
}
}

_ = Task.Run(() => PostBin(msg, attachment, fileContent));
if (!alreadyPostedBin)
_ = Task.Run(() => PostBin(msg, attachment, fileContent));
}
}
}

return botAnswer.ToString();
return botAnswer.ToString();
}
else
{
return "That file is quite large, if you want it to be scanned by CHEF for common log errors, please relaunch the game and exit as soon as you hit the main menu screen, it will prevent the file from getting a lot bigger than needed.";
}
}
}

Expand Down
41 changes: 40 additions & 1 deletion CHEF/Components/Watcher/CommonIssues.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
Expand Down Expand Up @@ -65,7 +66,7 @@ public static async Task<bool> CheckForOutdatedAndDeprecatedMods(string text, St
{
var textContainsAnyBadMod = false;

if (text != null)
if (!string.IsNullOrWhiteSpace(text))
{
const string ThunderstoreManifestPrefix = "TS Manifest: ";
if (text.Contains(ThunderstoreManifestPrefix, StringComparison.InvariantCultureIgnoreCase))
Expand Down Expand Up @@ -134,6 +135,44 @@ public static async Task<bool> CheckForOutdatedAndDeprecatedMods(string text, St
return textContainsAnyBadMod;
}

public static string RemoveDuplicateExceptionsFromText(string text)
{
if (!string.IsNullOrWhiteSpace(text))
{
using (var stringReader = new StringReader(text))
{
var isLinePartOfAnException = false;

HashSet<string> exceptions = new();
StringBuilder linesSb = new();

string line;
while ((line = stringReader.ReadLine()) != null)
{
if (line.Contains("Exception:", StringComparison.InvariantCulture))
{
isLinePartOfAnException = true;
}
else if (line.StartsWith('[') &&
line.Contains(':', StringComparison.InvariantCulture) &&
line.Contains(']', StringComparison.InvariantCulture))
{
isLinePartOfAnException = false;
}

if (!isLinePartOfAnException || exceptions.Add(line))
{
linesSb.AppendLine(line);
}
}

return linesSb.ToString();
}
}

return null;
}

/// <summary>
/// Check for the <paramref name="modName"/> if <paramref name="verFromText"/> is the latest version.<para/>
/// Returns the latest version as a string, null if <paramref name="verFromText"/> is the latest.
Expand Down

0 comments on commit 51a8b51

Please sign in to comment.