First attempt at streaming zip, doesn't work

Alas, this fails (when a ProjectController method is added to call
PrepareLdmlZip) with "System.InvalidOperationException: Synchronous
operations are disallowed. Call WriteAsync or set AllowSynchronousIO to
true instead."

We'll have to change this to prepare the entire zip file first, then
send it.

backend/LexBoxApi/Jobs/DelayedLexJob.cs

@@ -0,0 +1,28 @@
+using System.Diagnostics;
+using LexBoxApi.Otel;
+using OpenTelemetry.Trace;
+using Quartz;
+
+namespace LexBoxApi.Jobs;
+
+public abstract class DelayedLexJob() : LexJob
+{
+    protected static async Task QueueJob(ISchedulerFactory schedulerFactory,
+        JobKey key,
+        JobDataMap data,
+        TimeSpan delay,
+        CancellationToken cancellationToken = default)
+    {
+        var now = DateTime.UtcNow;
+        data[nameof(JobTriggerTraceId)] = Activity.Current?.Context.TraceId.ToHexString() ?? string.Empty;
+        data[nameof(JobTriggerSpanParentId)] = Activity.Current?.Context.SpanId.ToHexString() ?? string.Empty;
+        var trigger = TriggerBuilder.Create()
+            .WithIdentity(key.Name + "_Trigger", key.Group)
+            .StartAt(now.Add(delay))
+            .ForJob(key.Name, key.Group)
+            .UsingJobData(data)
+            .Build();
+        var scheduler = await schedulerFactory.GetScheduler(cancellationToken);
+        await scheduler.ScheduleJob(trigger, cancellationToken);
+    }
+}

backend/LexBoxApi/Jobs/DeleteTempDirectoryJob.cs

@@ -0,0 +1,30 @@
+using Quartz;
+
+namespace LexBoxApi.Jobs;
+
+public class DeleteTempDirectoryJob() : DelayedLexJob
+{
+    public static async Task Queue(ISchedulerFactory schedulerFactory,
+        string path,
+        TimeSpan delay,
+        CancellationToken cancellationToken = default)
+    {
+        await QueueJob(schedulerFactory,
+            Key,
+            new JobDataMap { { nameof(Path), path } },
+            delay,
+            cancellationToken);
+    }
+
+    public static JobKey Key { get; } = new(nameof(DeleteTempDirectoryJob), "CleanupJobs");
+    public string? Path { get; set; }
+
+    protected override Task ExecuteJob(IJobExecutionContext context)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(Path);
+        return Task.Run(() =>
+        {
+            if (Directory.Exists(Path)) Directory.Delete(Path, true);
+        });
+    }
+}

backend/LexBoxApi/ScheduledTasksKernel.cs

@@ -38,6 +38,7 @@ public static void AddScheduledTasks(this IServiceCollection services, IConfigur
 
             //Setup jobs
             q.AddJob<CleanupResetBackupJob>(CleanupResetBackupJob.Key);
+            q.AddJob<DeleteTempDirectoryJob>(DeleteTempDirectoryJob.Key, j => j.StoreDurably());
             q.AddJob<UpdateProjectMetadataJob>(UpdateProjectMetadataJob.Key, j => j.StoreDurably());
             q.AddJob<RetryEmailJob>(RetryEmailJob.Key, j => j.StoreDurably());
             q.AddTrigger(opts => opts.ForJob(CleanupResetBackupJob.Key)

backend/LexBoxApi/Services/HgService.cs

@@ -1,5 +1,6 @@
 using System.Diagnostics;
 using System.IO.Compression;
+using System.Net;
 using System.Net.Http.Headers;
 using System.Runtime.InteropServices;
 using System.Text;
@@ -491,6 +492,13 @@ public async Task<string> HgCommandHealth()
         return version.Trim();
     }
 
+    public async Task<ZipArchive?> GetLdmlZip(ProjectCode code, CancellationToken token = default)
+    {
+        var content = await ExecuteHgCommandServerCommand_ErrorsOk(code, "ldmlzip", [HttpStatusCode.Forbidden], token);
+        if (content is null) return null;
+        return new ZipArchive(await content.ReadAsStreamAsync(token), ZipArchiveMode.Read);
+    }
+
     private async Task<HttpContent> ExecuteHgCommandServerCommand(ProjectCode code, string command, CancellationToken token)
     {
         var httpClient = _hgClient.Value;
@@ -500,6 +508,16 @@ private async Task<HttpContent> ExecuteHgCommandServerCommand(ProjectCode code,
         return response.Content;
     }
 
+    private async Task<HttpContent?> ExecuteHgCommandServerCommand_ErrorsOk(ProjectCode code, string command, IEnumerable<HttpStatusCode> okErrors, CancellationToken token)
+    {
+        var httpClient = _hgClient.Value;
+        var baseUri = _options.Value.HgCommandServer;
+        var response = await httpClient.GetAsync($"{baseUri}{code}/{command}", HttpCompletionOption.ResponseHeadersRead, token);
+        if (okErrors.Contains(response.StatusCode)) return null;
+        response.EnsureSuccessStatusCode();
+        return response.Content;
+    }
+
     public async Task<ProjectType> DetermineProjectType(ProjectCode projectCode)
     {
         var response = await GetResponseMessage(projectCode, "file/tip?style=json-lex");

backend/LexBoxApi/Services/ProjectService.cs

@@ -1,4 +1,6 @@
 using System.Data.Common;
+using System.IO.Compression;
+using LexBoxApi.Jobs;
 using LexBoxApi.Models.Project;
 using LexBoxApi.Services.Email;
 using LexCore.Auth;
@@ -13,7 +15,7 @@
 
 namespace LexBoxApi.Services;
 
-public class ProjectService(LexBoxDbContext dbContext, IHgService hgService, IOptions<HgConfig> hgConfig, IMemoryCache memoryCache, IEmailService emailService)
+public class ProjectService(LexBoxDbContext dbContext, IHgService hgService, IOptions<HgConfig> hgConfig, IMemoryCache memoryCache, IEmailService emailService, Quartz.ISchedulerFactory schedulerFactory)
 {
     public async Task<Guid> CreateProject(CreateProjectInput input)
     {
@@ -269,6 +271,31 @@ public async Task ResetLexEntryCount(string projectCode)
         }
     }
 
+    public async Task<DirectoryInfo?> ExtractLdmlZip(Project project, string destRoot, CancellationToken token = default)
+    {
+        if (project.Type != ProjectType.FLEx) return null;
+        var zip = await hgService.GetLdmlZip(project.Code, token);
+        if (zip is null) return null;
+        var path = System.IO.Path.Join(destRoot, project.Id.ToString());
+        if (Directory.Exists(path)) Directory.Delete(path, true);
+        var dirInfo = Directory.CreateDirectory(path);
+        zip.ExtractToDirectory(dirInfo.FullName, true);
+        return dirInfo;
+    }
+
+    public async Task PrepareLdmlZip(Stream outStream, CancellationToken token = default)
+    {
+        var path = System.IO.Path.Join(System.IO.Path.GetTempPath(), "ldml-zip"); // TODO: pick random name, rather than predictable one
+        if (Directory.Exists(path)) Directory.Delete(path, true);
+        Directory.CreateDirectory(path);
+        await DeleteTempDirectoryJob.Queue(schedulerFactory, path, TimeSpan.FromHours(4));
+        await foreach (var project in dbContext.Projects.Where(p => p.Type == ProjectType.FLEx).AsAsyncEnumerable())
+        {
+            await ExtractLdmlZip(project, path, token);
+        }
+        ZipFile.CreateFromDirectory(path, outStream, CompressionLevel.Fastest, includeBaseDirectory: false);
+    }
+
     public async Task<DateTimeOffset?> UpdateLastCommit(string projectCode)
     {
         var project = await dbContext.Projects.FirstOrDefaultAsync(p => p.Code == projectCode);

backend/LexCore/ServiceInterfaces/IHgService.cs

@@ -22,6 +22,7 @@ public interface IHgService
     Task<int?> GetRepoSizeInKb(ProjectCode code, CancellationToken token = default);
     Task<int?> GetLexEntryCount(ProjectCode code, ProjectType projectType);
     Task<string?> GetRepositoryIdentifier(Project project);
+    Task<System.IO.Compression.ZipArchive?> GetLdmlZip(ProjectCode code, CancellationToken token = default);
     Task<HttpContent> ExecuteHgRecover(ProjectCode code, CancellationToken token);
     Task<HttpContent> InvalidateDirCache(ProjectCode code, CancellationToken token = default);
     bool HasAbandonedTransactions(ProjectCode projectCode);