From a39a21be40a884dd7e8c18d5a5e9ce2cc5a7ae71 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 15 Jan 2025 14:11:54 -0500 Subject: [PATCH] First attempt at streaming zip, doesn't work Alas, this fails (when a ProjectController method is added to call PrepareLdmlZip) with "System.InvalidOperationException: Synchronous operations are disallowed. Call WriteAsync or set AllowSynchronousIO to true instead." We'll have to change this to prepare the entire zip file first, then send it. --- backend/LexBoxApi/Jobs/DelayedLexJob.cs | 28 +++++++++++++++++ .../LexBoxApi/Jobs/DeleteTempDirectoryJob.cs | 30 +++++++++++++++++++ backend/LexBoxApi/ScheduledTasksKernel.cs | 1 + backend/LexBoxApi/Services/HgService.cs | 18 +++++++++++ backend/LexBoxApi/Services/ProjectService.cs | 29 +++++++++++++++++- .../LexCore/ServiceInterfaces/IHgService.cs | 1 + 6 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 backend/LexBoxApi/Jobs/DelayedLexJob.cs create mode 100644 backend/LexBoxApi/Jobs/DeleteTempDirectoryJob.cs diff --git a/backend/LexBoxApi/Jobs/DelayedLexJob.cs b/backend/LexBoxApi/Jobs/DelayedLexJob.cs new file mode 100644 index 000000000..25060d61b --- /dev/null +++ b/backend/LexBoxApi/Jobs/DelayedLexJob.cs @@ -0,0 +1,28 @@ +using System.Diagnostics; +using LexBoxApi.Otel; +using OpenTelemetry.Trace; +using Quartz; + +namespace LexBoxApi.Jobs; + +public abstract class DelayedLexJob() : LexJob +{ + protected static async Task QueueJob(ISchedulerFactory schedulerFactory, + JobKey key, + JobDataMap data, + TimeSpan delay, + CancellationToken cancellationToken = default) + { + var now = DateTime.UtcNow; + data[nameof(JobTriggerTraceId)] = Activity.Current?.Context.TraceId.ToHexString() ?? string.Empty; + data[nameof(JobTriggerSpanParentId)] = Activity.Current?.Context.SpanId.ToHexString() ?? string.Empty; + var trigger = TriggerBuilder.Create() + .WithIdentity(key.Name + "_Trigger", key.Group) + .StartAt(now.Add(delay)) + .ForJob(key.Name, key.Group) + .UsingJobData(data) + .Build(); + var scheduler = await schedulerFactory.GetScheduler(cancellationToken); + await scheduler.ScheduleJob(trigger, cancellationToken); + } +} diff --git a/backend/LexBoxApi/Jobs/DeleteTempDirectoryJob.cs b/backend/LexBoxApi/Jobs/DeleteTempDirectoryJob.cs new file mode 100644 index 000000000..3c8306349 --- /dev/null +++ b/backend/LexBoxApi/Jobs/DeleteTempDirectoryJob.cs @@ -0,0 +1,30 @@ +using Quartz; + +namespace LexBoxApi.Jobs; + +public class DeleteTempDirectoryJob() : DelayedLexJob +{ + public static async Task Queue(ISchedulerFactory schedulerFactory, + string path, + TimeSpan delay, + CancellationToken cancellationToken = default) + { + await QueueJob(schedulerFactory, + Key, + new JobDataMap { { nameof(Path), path } }, + delay, + cancellationToken); + } + + public static JobKey Key { get; } = new(nameof(DeleteTempDirectoryJob), "CleanupJobs"); + public string? Path { get; set; } + + protected override Task ExecuteJob(IJobExecutionContext context) + { + ArgumentException.ThrowIfNullOrEmpty(Path); + return Task.Run(() => + { + if (Directory.Exists(Path)) Directory.Delete(Path, true); + }); + } +} diff --git a/backend/LexBoxApi/ScheduledTasksKernel.cs b/backend/LexBoxApi/ScheduledTasksKernel.cs index 572cc7b35..45372a712 100644 --- a/backend/LexBoxApi/ScheduledTasksKernel.cs +++ b/backend/LexBoxApi/ScheduledTasksKernel.cs @@ -38,6 +38,7 @@ public static void AddScheduledTasks(this IServiceCollection services, IConfigur //Setup jobs q.AddJob(CleanupResetBackupJob.Key); + q.AddJob(DeleteTempDirectoryJob.Key, j => j.StoreDurably()); q.AddJob(UpdateProjectMetadataJob.Key, j => j.StoreDurably()); q.AddJob(RetryEmailJob.Key, j => j.StoreDurably()); q.AddTrigger(opts => opts.ForJob(CleanupResetBackupJob.Key) diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs index 32e328943..8798dc518 100644 --- a/backend/LexBoxApi/Services/HgService.cs +++ b/backend/LexBoxApi/Services/HgService.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.IO.Compression; +using System.Net; using System.Net.Http.Headers; using System.Runtime.InteropServices; using System.Text; @@ -491,6 +492,13 @@ public async Task HgCommandHealth() return version.Trim(); } + public async Task GetLdmlZip(ProjectCode code, CancellationToken token = default) + { + var content = await ExecuteHgCommandServerCommand_ErrorsOk(code, "ldmlzip", [HttpStatusCode.Forbidden], token); + if (content is null) return null; + return new ZipArchive(await content.ReadAsStreamAsync(token), ZipArchiveMode.Read); + } + private async Task ExecuteHgCommandServerCommand(ProjectCode code, string command, CancellationToken token) { var httpClient = _hgClient.Value; @@ -500,6 +508,16 @@ private async Task ExecuteHgCommandServerCommand(ProjectCode code, return response.Content; } + private async Task ExecuteHgCommandServerCommand_ErrorsOk(ProjectCode code, string command, IEnumerable okErrors, CancellationToken token) + { + var httpClient = _hgClient.Value; + var baseUri = _options.Value.HgCommandServer; + var response = await httpClient.GetAsync($"{baseUri}{code}/{command}", HttpCompletionOption.ResponseHeadersRead, token); + if (okErrors.Contains(response.StatusCode)) return null; + response.EnsureSuccessStatusCode(); + return response.Content; + } + public async Task DetermineProjectType(ProjectCode projectCode) { var response = await GetResponseMessage(projectCode, "file/tip?style=json-lex"); diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index 095f43582..deb983c89 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -1,4 +1,6 @@ using System.Data.Common; +using System.IO.Compression; +using LexBoxApi.Jobs; using LexBoxApi.Models.Project; using LexBoxApi.Services.Email; using LexCore.Auth; @@ -13,7 +15,7 @@ namespace LexBoxApi.Services; -public class ProjectService(LexBoxDbContext dbContext, IHgService hgService, IOptions hgConfig, IMemoryCache memoryCache, IEmailService emailService) +public class ProjectService(LexBoxDbContext dbContext, IHgService hgService, IOptions hgConfig, IMemoryCache memoryCache, IEmailService emailService, Quartz.ISchedulerFactory schedulerFactory) { public async Task CreateProject(CreateProjectInput input) { @@ -269,6 +271,31 @@ public async Task ResetLexEntryCount(string projectCode) } } + public async Task ExtractLdmlZip(Project project, string destRoot, CancellationToken token = default) + { + if (project.Type != ProjectType.FLEx) return null; + var zip = await hgService.GetLdmlZip(project.Code, token); + if (zip is null) return null; + var path = System.IO.Path.Join(destRoot, project.Id.ToString()); + if (Directory.Exists(path)) Directory.Delete(path, true); + var dirInfo = Directory.CreateDirectory(path); + zip.ExtractToDirectory(dirInfo.FullName, true); + return dirInfo; + } + + public async Task PrepareLdmlZip(Stream outStream, CancellationToken token = default) + { + var path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "ldml-zip"); // TODO: pick random name, rather than predictable one + if (Directory.Exists(path)) Directory.Delete(path, true); + Directory.CreateDirectory(path); + await DeleteTempDirectoryJob.Queue(schedulerFactory, path, TimeSpan.FromHours(4)); + await foreach (var project in dbContext.Projects.Where(p => p.Type == ProjectType.FLEx).AsAsyncEnumerable()) + { + await ExtractLdmlZip(project, path, token); + } + ZipFile.CreateFromDirectory(path, outStream, CompressionLevel.Fastest, includeBaseDirectory: false); + } + public async Task UpdateLastCommit(string projectCode) { var project = await dbContext.Projects.FirstOrDefaultAsync(p => p.Code == projectCode); diff --git a/backend/LexCore/ServiceInterfaces/IHgService.cs b/backend/LexCore/ServiceInterfaces/IHgService.cs index d21347e9c..e3d513ac4 100644 --- a/backend/LexCore/ServiceInterfaces/IHgService.cs +++ b/backend/LexCore/ServiceInterfaces/IHgService.cs @@ -22,6 +22,7 @@ public interface IHgService Task GetRepoSizeInKb(ProjectCode code, CancellationToken token = default); Task GetLexEntryCount(ProjectCode code, ProjectType projectType); Task GetRepositoryIdentifier(Project project); + Task GetLdmlZip(ProjectCode code, CancellationToken token = default); Task ExecuteHgRecover(ProjectCode code, CancellationToken token); Task InvalidateDirCache(ProjectCode code, CancellationToken token = default); bool HasAbandonedTransactions(ProjectCode projectCode);