diff --git a/backend/LexBoxApi/Jobs/DelayedLexJob.cs b/backend/LexBoxApi/Jobs/DelayedLexJob.cs new file mode 100644 index 000000000..25060d61b --- /dev/null +++ b/backend/LexBoxApi/Jobs/DelayedLexJob.cs @@ -0,0 +1,28 @@ +using System.Diagnostics; +using LexBoxApi.Otel; +using OpenTelemetry.Trace; +using Quartz; + +namespace LexBoxApi.Jobs; + +public abstract class DelayedLexJob() : LexJob +{ + protected static async Task QueueJob(ISchedulerFactory schedulerFactory, + JobKey key, + JobDataMap data, + TimeSpan delay, + CancellationToken cancellationToken = default) + { + var now = DateTime.UtcNow; + data[nameof(JobTriggerTraceId)] = Activity.Current?.Context.TraceId.ToHexString() ?? string.Empty; + data[nameof(JobTriggerSpanParentId)] = Activity.Current?.Context.SpanId.ToHexString() ?? string.Empty; + var trigger = TriggerBuilder.Create() + .WithIdentity(key.Name + "_Trigger", key.Group) + .StartAt(now.Add(delay)) + .ForJob(key.Name, key.Group) + .UsingJobData(data) + .Build(); + var scheduler = await schedulerFactory.GetScheduler(cancellationToken); + await scheduler.ScheduleJob(trigger, cancellationToken); + } +} diff --git a/backend/LexBoxApi/Jobs/DeleteTempDirectoryJob.cs b/backend/LexBoxApi/Jobs/DeleteTempDirectoryJob.cs new file mode 100644 index 000000000..3c8306349 --- /dev/null +++ b/backend/LexBoxApi/Jobs/DeleteTempDirectoryJob.cs @@ -0,0 +1,30 @@ +using Quartz; + +namespace LexBoxApi.Jobs; + +public class DeleteTempDirectoryJob() : DelayedLexJob +{ + public static async Task Queue(ISchedulerFactory schedulerFactory, + string path, + TimeSpan delay, + CancellationToken cancellationToken = default) + { + await QueueJob(schedulerFactory, + Key, + new JobDataMap { { nameof(Path), path } }, + delay, + cancellationToken); + } + + public static JobKey Key { get; } = new(nameof(DeleteTempDirectoryJob), "CleanupJobs"); + public string? Path { get; set; } + + protected override Task ExecuteJob(IJobExecutionContext context) + { + ArgumentException.ThrowIfNullOrEmpty(Path); + return Task.Run(() => + { + if (Directory.Exists(Path)) Directory.Delete(Path, true); + }); + } +} diff --git a/backend/LexBoxApi/ScheduledTasksKernel.cs b/backend/LexBoxApi/ScheduledTasksKernel.cs index 572cc7b35..45372a712 100644 --- a/backend/LexBoxApi/ScheduledTasksKernel.cs +++ b/backend/LexBoxApi/ScheduledTasksKernel.cs @@ -38,6 +38,7 @@ public static void AddScheduledTasks(this IServiceCollection services, IConfigur //Setup jobs q.AddJob(CleanupResetBackupJob.Key); + q.AddJob(DeleteTempDirectoryJob.Key, j => j.StoreDurably()); q.AddJob(UpdateProjectMetadataJob.Key, j => j.StoreDurably()); q.AddJob(RetryEmailJob.Key, j => j.StoreDurably()); q.AddTrigger(opts => opts.ForJob(CleanupResetBackupJob.Key) diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs index 32e328943..8798dc518 100644 --- a/backend/LexBoxApi/Services/HgService.cs +++ b/backend/LexBoxApi/Services/HgService.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.IO.Compression; +using System.Net; using System.Net.Http.Headers; using System.Runtime.InteropServices; using System.Text; @@ -491,6 +492,13 @@ public async Task HgCommandHealth() return version.Trim(); } + public async Task GetLdmlZip(ProjectCode code, CancellationToken token = default) + { + var content = await ExecuteHgCommandServerCommand_ErrorsOk(code, "ldmlzip", [HttpStatusCode.Forbidden], token); + if (content is null) return null; + return new ZipArchive(await content.ReadAsStreamAsync(token), ZipArchiveMode.Read); + } + private async Task ExecuteHgCommandServerCommand(ProjectCode code, string command, CancellationToken token) { var httpClient = _hgClient.Value; @@ -500,6 +508,16 @@ private async Task ExecuteHgCommandServerCommand(ProjectCode code, return response.Content; } + private async Task ExecuteHgCommandServerCommand_ErrorsOk(ProjectCode code, string command, IEnumerable okErrors, CancellationToken token) + { + var httpClient = _hgClient.Value; + var baseUri = _options.Value.HgCommandServer; + var response = await httpClient.GetAsync($"{baseUri}{code}/{command}", HttpCompletionOption.ResponseHeadersRead, token); + if (okErrors.Contains(response.StatusCode)) return null; + response.EnsureSuccessStatusCode(); + return response.Content; + } + public async Task DetermineProjectType(ProjectCode projectCode) { var response = await GetResponseMessage(projectCode, "file/tip?style=json-lex"); diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index 095f43582..deb983c89 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -1,4 +1,6 @@ using System.Data.Common; +using System.IO.Compression; +using LexBoxApi.Jobs; using LexBoxApi.Models.Project; using LexBoxApi.Services.Email; using LexCore.Auth; @@ -13,7 +15,7 @@ namespace LexBoxApi.Services; -public class ProjectService(LexBoxDbContext dbContext, IHgService hgService, IOptions hgConfig, IMemoryCache memoryCache, IEmailService emailService) +public class ProjectService(LexBoxDbContext dbContext, IHgService hgService, IOptions hgConfig, IMemoryCache memoryCache, IEmailService emailService, Quartz.ISchedulerFactory schedulerFactory) { public async Task CreateProject(CreateProjectInput input) { @@ -269,6 +271,31 @@ public async Task ResetLexEntryCount(string projectCode) } } + public async Task ExtractLdmlZip(Project project, string destRoot, CancellationToken token = default) + { + if (project.Type != ProjectType.FLEx) return null; + var zip = await hgService.GetLdmlZip(project.Code, token); + if (zip is null) return null; + var path = System.IO.Path.Join(destRoot, project.Id.ToString()); + if (Directory.Exists(path)) Directory.Delete(path, true); + var dirInfo = Directory.CreateDirectory(path); + zip.ExtractToDirectory(dirInfo.FullName, true); + return dirInfo; + } + + public async Task PrepareLdmlZip(Stream outStream, CancellationToken token = default) + { + var path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "ldml-zip"); // TODO: pick random name, rather than predictable one + if (Directory.Exists(path)) Directory.Delete(path, true); + Directory.CreateDirectory(path); + await DeleteTempDirectoryJob.Queue(schedulerFactory, path, TimeSpan.FromHours(4)); + await foreach (var project in dbContext.Projects.Where(p => p.Type == ProjectType.FLEx).AsAsyncEnumerable()) + { + await ExtractLdmlZip(project, path, token); + } + ZipFile.CreateFromDirectory(path, outStream, CompressionLevel.Fastest, includeBaseDirectory: false); + } + public async Task UpdateLastCommit(string projectCode) { var project = await dbContext.Projects.FirstOrDefaultAsync(p => p.Code == projectCode); diff --git a/backend/LexCore/ServiceInterfaces/IHgService.cs b/backend/LexCore/ServiceInterfaces/IHgService.cs index d21347e9c..e3d513ac4 100644 --- a/backend/LexCore/ServiceInterfaces/IHgService.cs +++ b/backend/LexCore/ServiceInterfaces/IHgService.cs @@ -22,6 +22,7 @@ public interface IHgService Task GetRepoSizeInKb(ProjectCode code, CancellationToken token = default); Task GetLexEntryCount(ProjectCode code, ProjectType projectType); Task GetRepositoryIdentifier(Project project); + Task GetLdmlZip(ProjectCode code, CancellationToken token = default); Task ExecuteHgRecover(ProjectCode code, CancellationToken token); Task InvalidateDirCache(ProjectCode code, CancellationToken token = default); bool HasAbandonedTransactions(ProjectCode projectCode);