Add merge status API to fw-headless (#1294)

Creates a new GET /api/crdt-sync-status endpoint.

Returns 404 if the project ID does not match any project in the lexbox
database. For all other projects, this returns Syncing if a CRDT sync is
currently in progress, NeverSynced if there is no CRDT project yet, and
ReadyToSync if the CRDT project exists but is not currently syncing. If
the status is ReadyToSync, it will also return a count of commits
available in the Lexbox DB that are not yet present in the local CRDT
database. (The count is naive, just doing (lexbox commits - local
commits), so if the local DB has commits not present in Lexbox the count
could even end up negative.)

backend/FwHeadless/FwHeadlessKernel.cs

@@ -17,6 +17,7 @@ public static void AddFwHeadless(this IServiceCollection services)
             .BindConfiguration("FwHeadlessConfig")
             .ValidateDataAnnotations()
             .ValidateOnStart();
+        services.AddSingleton<SyncJobStatusService>();
         services.AddScoped<SendReceiveService>();
         services.AddScoped<ProjectLookupService>();
         services.AddScoped<LogSanitizerService>();
@@ -26,6 +27,7 @@ public static void AddFwHeadless(this IServiceCollection services)
             .AddFwDataBridge()
             .AddFwLiteProjectSync();
         services.AddScoped<CrdtSyncService>();
+        services.AddScoped<ProjectContextFromIdService>();
         services.AddTransient<HttpClientAuthHandler>();
         services.AddHttpClient(LexboxHttpClientName,
             (provider, client) =>

backend/FwHeadless/Program.cs

@@ -1,4 +1,5 @@
 using FwHeadless;
+using FwHeadless.Services;
 using FwDataMiniLcmBridge;
 using FwDataMiniLcmBridge.Api;
 using FwLiteProjectSync;
@@ -10,6 +11,10 @@
 using Microsoft.Extensions.Options;
 using MiniLcm;
 using Scalar.AspNetCore;
+using LexCore.Utils;
+using LinqToDB;
+using SIL.Harmony.Core;
+using SIL.Harmony;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -36,6 +41,13 @@
     await next();
 });
 
+// Load project ID from request
+app.Use((context, next) =>
+{
+    var renameThisService = context.RequestServices.GetRequiredService<ProjectContextFromIdService>();
+    return renameThisService.PopulateProjectContext(context, next);
+});
+
 // Configure the HTTP request pipeline.
 if (app.Environment.IsDevelopment())
 {
@@ -49,6 +61,7 @@
 app.MapHealthChecks("/api/healthz");
 
 app.MapPost("/api/crdt-sync", ExecuteMergeRequest);
+app.MapGet("/api/crdt-sync-status", GetMergeStatus);
 
 app.Run();
 
@@ -60,6 +73,7 @@ static async Task<Results<Ok<SyncResult>, NotFound, ProblemHttpResult>> ExecuteM
     FwDataFactory fwDataFactory,
     CrdtProjectsService projectsService,
     ProjectLookupService projectLookupService,
+    SyncJobStatusService syncStatusService,
     CrdtFwdataProjectSyncService syncService,
     CrdtHttpSyncService crdtHttpSyncService,
     IHttpClientFactory httpClientFactory,
@@ -73,6 +87,9 @@ static async Task<Results<Ok<SyncResult>, NotFound, ProblemHttpResult>> ExecuteM
         return TypedResults.Ok(new SyncResult(0, 0));
     }
 
+    syncStatusService.StartSyncing(projectId);
+    using var stopSyncing = Defer.Action(() => syncStatusService.StopSyncing(projectId));
+
     var projectCode = await projectLookupService.GetProjectCode(projectId);
     if (projectCode is null)
     {
@@ -103,7 +120,6 @@ static async Task<Results<Ok<SyncResult>, NotFound, ProblemHttpResult>> ExecuteM
     var crdtSyncService = services.GetRequiredService<CrdtSyncService>();
     await crdtSyncService.Sync();
 
-
     var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun);
     logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges);
 
@@ -113,6 +129,29 @@ static async Task<Results<Ok<SyncResult>, NotFound, ProblemHttpResult>> ExecuteM
     return TypedResults.Ok(result);
 }
 
+static async Task<Results<Ok<ProjectSyncStatus>, NotFound>> GetMergeStatus(
+    ProjectContext projectContext,
+    ProjectLookupService projectLookupService,
+    SyncJobStatusService syncJobStatusService,
+    IServiceProvider services,
+    LexBoxDbContext lexBoxDb,
+    Guid projectId)
+{
+    var jobStatus = syncJobStatusService.SyncStatus(projectId);
+    if (jobStatus == SyncJobStatus.Running) return TypedResults.Ok(ProjectSyncStatus.Syncing);
+    var project = projectContext.Project;
+    if (project is null)
+    {
+        // 404 only means "project doesn't exist"; if we don't know the status, then it hasn't synced before and is therefore ready to sync
+        if (await projectLookupService.ProjectExists(projectId)) return TypedResults.Ok(ProjectSyncStatus.NeverSynced);
+        else return TypedResults.NotFound();
+    }
+    var commitsOnServer = await lexBoxDb.Set<ServerCommit>().CountAsync(c => c.ProjectId == projectId);
+    var lcmCrdtDbContext = services.GetRequiredService<LcmCrdtDbContext>();
+    var localCommits = await lcmCrdtDbContext.Set<Commit>().CountAsync();
+    return TypedResults.Ok(ProjectSyncStatus.ReadyToSync(commitsOnServer - localCommits));
+}
+
 static async Task<FwDataMiniLcmApi> SetupFwData(FwDataProject fwDataProject,
     SendReceiveService srService,
     string projectCode,

backend/FwHeadless/Services/AppVersionService.cs

@@ -1,6 +1,6 @@
 using System.Reflection;
 
-namespace FwHeadless;
+namespace FwHeadless.Services;
 
 public static class AppVersionService
 {

backend/FwHeadless/CrdtSyncService.cs → backend/FwHeadless/Services/CrdtSyncService.cs

@@ -2,7 +2,7 @@
 using LcmCrdt.RemoteSync;
 using SIL.Harmony;
 
-namespace FwHeadless;
+namespace FwHeadless.Services;
 
 public class CrdtSyncService(
     CrdtHttpSyncService httpSyncService,

backend/FwHeadless/Services/HttpHelpers.cs

@@ -0,0 +1,14 @@
+namespace FwHeadless.Services;
+
+public static class HttpHelpers
+{
+    public static Guid? GetProjectId(this HttpContext? context)
+    {
+        if (context is null) return null;
+        if (context.Request.Query.TryGetValue("projectId", out var projectIds) && projectIds.FirstOrDefault() is string idStr)
+        {
+            if (Guid.TryParse(idStr, out var projectId)) return projectId;
+        }
+        return null;
+    }
+}

backend/FwHeadless/Services/ProjectContextFromIdService.cs

@@ -0,0 +1,28 @@
+using LcmCrdt;
+using Microsoft.Extensions.Options;
+
+namespace FwHeadless.Services;
+
+// TODO: Pick better name
+public class ProjectContextFromIdService(IOptions<FwHeadlessConfig> config, CrdtProjectsService projectsService, ProjectLookupService projectLookupService)
+{
+    public async Task PopulateProjectContext(HttpContext context, Func<Task> next)
+    {
+        if (context.GetProjectId() is Guid projectId)
+        {
+            var projectCode = await projectLookupService.GetProjectCode(projectId);
+            if (projectCode is not null)
+            {
+                var projectFolder = Path.Join(config.Value.ProjectStorageRoot, $"{projectCode}-{projectId}");
+                var crdtFile = Path.Join(projectFolder, "crdt.sqlite");
+                if (File.Exists(crdtFile))
+                {
+                    var project = new CrdtProject("crdt", crdtFile);
+                    projectsService.SetProjectScope(project);
+                    await context.RequestServices.GetRequiredService<CurrentProjectService>().PopulateProjectDataCache();
+                }
+            }
+        }
+        await next();
+    }
+}

backend/FwHeadless/ProjectLookupService.cs → backend/FwHeadless/Services/ProjectLookupService.cs

@@ -2,7 +2,7 @@
 using Microsoft.EntityFrameworkCore;
 using SIL.Harmony.Core;
 
-namespace FwHeadless;
+namespace FwHeadless.Services;
 
 public class ProjectLookupService(LexBoxDbContext dbContext)
 {
@@ -15,6 +15,11 @@ public class ProjectLookupService(LexBoxDbContext dbContext)
         return projectCode;
     }
 
+    public async Task<bool> ProjectExists(Guid projectId)
+    {
+        return await dbContext.Projects.AnyAsync(p => p.Id == projectId);
+    }
+
     public async Task<bool> IsCrdtProject(Guid projectId)
     {
         return await dbContext.Set<ServerCommit>().AnyAsync(c => c.ProjectId == projectId);

backend/FwHeadless/Services/ProjectSyncStatusService.cs

@@ -0,0 +1,49 @@
+using System.Collections.Concurrent;
+
+namespace FwHeadless.Services;
+
+public class SyncJobStatusService()
+{
+    private ConcurrentDictionary<Guid, SyncJobStatus> Status { get; init; } = new();
+
+    public void StartSyncing(Guid projectId)
+    {
+        Status.AddOrUpdate(projectId, (_) => SyncJobStatus.Running, (_, _) => SyncJobStatus.Running);
+    }
+
+    public void StopSyncing(Guid projectId)
+    {
+        Status.Remove(projectId, out var _);
+    }
+
+    public SyncJobStatus SyncStatus(Guid projectId)
+    {
+        return Status.TryGetValue(projectId, out var status) ? status : SyncJobStatus.NotRunning;
+    }
+}
+
+public enum SyncJobStatus
+{
+    NotRunning,
+    Running,
+}
+
+public enum ProjectSyncStatusEnum
+{
+    NeverSynced,
+    ReadyToSync,
+    Syncing,
+}
+
+// TODO: Bikeshed this name
+public record ProjectSyncStatus(
+    ProjectSyncStatusEnum status,
+    int ChangesAvailable)
+{
+    public static ProjectSyncStatus NeverSynced => new(ProjectSyncStatusEnum.NeverSynced, 0);
+    public static ProjectSyncStatus Syncing => new(ProjectSyncStatusEnum.Syncing, 0);
+    public static ProjectSyncStatus ReadyToSync(int changes)
+    {
+        return new(ProjectSyncStatusEnum.ReadyToSync, changes);
+    }
+}

backend/FwHeadless/SendReceiveHelpers.cs → backend/FwHeadless/Services/SendReceiveHelpers.cs

@@ -1,7 +1,7 @@
 using FwDataMiniLcmBridge;
 using SIL.Progress;
 
-namespace FwHeadless;
+namespace FwHeadless.Services;
 
 public static class SendReceiveHelpers
 {

backend/FwHeadless/SendReceiveService.cs → backend/FwHeadless/Services/SendReceiveService.cs

@@ -1,8 +1,7 @@
 using FwDataMiniLcmBridge;
-using FwHeadless.Services;
 using Microsoft.Extensions.Options;
 
-namespace FwHeadless;
+namespace FwHeadless.Services;
 
 public class SendReceiveService(IOptions<FwHeadlessConfig> config, SafeLoggingProgress progress)
 {