Skip to content

Commit

Permalink
Handle temporary files being present when loading guild data (#345)
Browse files Browse the repository at this point in the history 
This PR fixes catastrophic guild data loading errors that appear when
there are lingering temporary files. In normal operation, temporary
files are deleted as soon as they are copied to the main file. It is
also expected that temporary files are valid JSON files.

However, due to a yesterday's DoS attack, something:tm: happened and a
bunch of empty temporary files got written to disk. When Octobot
recovered from the attack, it was unable to load any guild data because
of the temporary files.

This PR addresses this issue by changing the data loading logic:
1) Check if there's a temporary file. If it exists, try loading it.
2) If it is successfully loaded, move the temp file to the main file and
resume operation as normal
3) If it could not be loaded, try loading the main file
4) If it is successfully loaded, delete the temporary file and resume
operation as normal
5) If it is not, throw an error (like before)

This PR was tested on production data and managed to load every guild
without errors.

Signed-off-by: Octol1ttle <[email protected]>
  • Loading branch information
@Octol1ttle
Octol1ttle authored Feb 3, 2025
1 parent bf81840 commit 4785d16
Show file tree
Hide file tree
Showing 2 changed files with 139 additions and 42 deletions.
2 changes: 1 addition & 1 deletion TeamOctolings.Octobot/Extensions/GuildScheduledEventExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ public static Result TryGetExternalEventData(this IGuildScheduledEvent scheduled
out string? location)
{
endTime = default;
location = default;
location = null;
if (!scheduledEvent.EntityMetadata.AsOptional().IsDefined(out var metadata))
{
return new ArgumentNullError(nameof(scheduledEvent.EntityMetadata));
Expand Down
179 changes: 138 additions & 41 deletions TeamOctolings.Octobot/Services/GuildDataService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,89 +75,186 @@ private async Task<GuildData> InitializeData(Snowflake guildId, CancellationToke
{
var path = $"GuildData/{guildId}";
var memberDataPath = $"{path}/MemberData";

var settingsPath = $"{path}/Settings.json";

var scheduledEventsPath = $"{path}/ScheduledEvents.json";

MigrateDataDirectory(guildId, path);

Directory.CreateDirectory(path);

if (!File.Exists(settingsPath))
var dataLoadFailed = false;

var jsonSettings = await LoadGuildSettings(settingsPath, ct);
if (jsonSettings is not null)
{
await File.WriteAllTextAsync(settingsPath, "{}", ct);
FixJsonSettings(jsonSettings);
}
else
{
dataLoadFailed = true;
}

if (!File.Exists(scheduledEventsPath))
var events = await LoadScheduledEvents(scheduledEventsPath, ct);
if (events is null)
{
await File.WriteAllTextAsync(scheduledEventsPath, "{}", ct);
dataLoadFailed = true;
}

var dataLoadFailed = false;
var memberData = new Dictionary<ulong, MemberData>();
foreach (var dataFileInfo in Directory.CreateDirectory(memberDataPath).GetFiles()
.Where(dataFileInfo =>
!memberData.ContainsKey(
ulong.Parse(dataFileInfo.Name.Replace(".json", "").Replace(".tmp", "")))))
{
var data = await LoadMemberData(dataFileInfo, memberDataPath, true, ct);

if (data == null)
{
dataLoadFailed = true;
continue;
}

memberData.TryAdd(data.Id, data);
}

var finalData = new GuildData(
jsonSettings ?? new JsonObject(), settingsPath,
events ?? new Dictionary<ulong, ScheduledEventData>(), scheduledEventsPath,
memberData, memberDataPath,
dataLoadFailed);

_datas.TryAdd(guildId, finalData);

return finalData;
}

private async Task<MemberData?> LoadMemberData(FileInfo dataFileInfo, string memberDataPath, bool loadTmp,
CancellationToken ct = default)
{
MemberData? data;
var temporaryPath = $"{dataFileInfo.FullName}.tmp";
var usedInfo = loadTmp && File.Exists(temporaryPath) ? new FileInfo(temporaryPath) : dataFileInfo;

await using var settingsStream = File.OpenRead(settingsPath);
JsonNode? jsonSettings = null;
var isTmp = usedInfo.Extension is ".tmp";
try
{
jsonSettings = await JsonNode.ParseAsync(settingsStream, cancellationToken: ct);
await using var dataStream = usedInfo.OpenRead();
data = await JsonSerializer.DeserializeAsync<MemberData>(dataStream, cancellationToken: ct);
if (isTmp)
{
usedInfo.CopyTo(usedInfo.FullName.Replace(".tmp", ""), true);
usedInfo.Delete();
}
}
catch (Exception e)
{
_logger.LogError(e, "Guild settings load failed: {Path}", settingsPath);
dataLoadFailed = true;
if (isTmp)
{
_logger.LogWarning(e,
"Unable to load temporary member data file, deleting: {MemberDataPath}/{FileName}", memberDataPath,
usedInfo.Name);
usedInfo.Delete();
return await LoadMemberData(dataFileInfo, memberDataPath, false, ct);
}

_logger.LogError(e, "Member data load failed: {MemberDataPath}/{FileName}", memberDataPath,
usedInfo.Name);
return null;
}

if (jsonSettings is not null)
return data;
}

private async Task<Dictionary<ulong, ScheduledEventData>?> LoadScheduledEvents(string scheduledEventsPath,
CancellationToken ct = default)
{
var tempScheduledEventsPath = $"{scheduledEventsPath}.tmp";

if (!File.Exists(scheduledEventsPath) && !File.Exists(tempScheduledEventsPath))
{
FixJsonSettings(jsonSettings);
return new Dictionary<ulong, ScheduledEventData>();
}

if (File.Exists(tempScheduledEventsPath))
{
_logger.LogWarning("Found temporary scheduled events file, will try to parse and copy to main: ${Path}",
tempScheduledEventsPath);
try
{
await using var tempEventsStream = File.OpenRead(tempScheduledEventsPath);
var events = await JsonSerializer.DeserializeAsync<Dictionary<ulong, ScheduledEventData>>(
tempEventsStream, cancellationToken: ct);
File.Copy(tempScheduledEventsPath, scheduledEventsPath, true);
File.Delete(tempScheduledEventsPath);

_logger.LogInformation("Successfully loaded temporary scheduled events file: ${Path}",
tempScheduledEventsPath);
return events;
}
catch (Exception e)
{
_logger.LogError(e, "Unable to load temporary scheduled events file: {Path}, deleting",
tempScheduledEventsPath);
File.Delete(tempScheduledEventsPath);
}
}

await using var eventsStream = File.OpenRead(scheduledEventsPath);
Dictionary<ulong, ScheduledEventData>? events = null;
try
{
events = await JsonSerializer.DeserializeAsync<Dictionary<ulong, ScheduledEventData>>(
await using var eventsStream = File.OpenRead(scheduledEventsPath);
return await JsonSerializer.DeserializeAsync<Dictionary<ulong, ScheduledEventData>>(
eventsStream, cancellationToken: ct);
}
catch (Exception e)
{
_logger.LogError(e, "Guild scheduled events load failed: {Path}", scheduledEventsPath);
dataLoadFailed = true;
return null;
}
}

var memberData = new Dictionary<ulong, MemberData>();
foreach (var dataFileInfo in Directory.CreateDirectory(memberDataPath).GetFiles())
private async Task<JsonNode?> LoadGuildSettings(string settingsPath, CancellationToken ct = default)
{
var tempSettingsPath = $"{settingsPath}.tmp";

if (!File.Exists(settingsPath) && !File.Exists(tempSettingsPath))
{
await using var dataStream = dataFileInfo.OpenRead();
MemberData? data;
return new JsonObject();
}

if (File.Exists(tempSettingsPath))
{
_logger.LogWarning("Found temporary settings file, will try to parse and copy to main: ${Path}",
tempSettingsPath);
try
{
data = await JsonSerializer.DeserializeAsync<MemberData>(dataStream, cancellationToken: ct);
await using var tempSettingsStream = File.OpenRead(tempSettingsPath);
var jsonSettings = await JsonNode.ParseAsync(tempSettingsStream, cancellationToken: ct);

File.Copy(tempSettingsPath, settingsPath, true);
File.Delete(tempSettingsPath);

_logger.LogInformation("Successfully loaded temporary settings file: ${Path}", tempSettingsPath);
return jsonSettings;
}
catch (Exception e)
{
_logger.LogError(e, "Member data load failed: {MemberDataPath}/{FileName}", memberDataPath,
dataFileInfo.Name);
dataLoadFailed = true;
continue;
}

if (data is null)
{
continue;
_logger.LogError(e, "Unable to load temporary settings file: {Path}, deleting", tempSettingsPath);
File.Delete(tempSettingsPath);
}

memberData.Add(data.Id, data);
}

var finalData = new GuildData(
jsonSettings ?? new JsonObject(), settingsPath,
events ?? new Dictionary<ulong, ScheduledEventData>(), scheduledEventsPath,
memberData, memberDataPath,
dataLoadFailed);

_datas.TryAdd(guildId, finalData);

return finalData;
try
{
await using var settingsStream = File.OpenRead(settingsPath);
return await JsonNode.ParseAsync(settingsStream, cancellationToken: ct);
}
catch (Exception e)
{
_logger.LogError(e, "Guild settings load failed: {Path}", settingsPath);
return null;
}
}

private void MigrateDataDirectory(Snowflake guildId, string newPath)
Expand Down

0 comments on commit 4785d16

Please sign in to comment.