Merge branch 'develop' into minilcm-diff-apis

# Conflicts:
#	backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs
#	backend/FwLite/FwLiteProjectSync.Tests/Fixtures/SyncFixture.cs
#	backend/FwLite/FwLiteProjectSync.Tests/UpdateDiffTests.cs
#	backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs
#	backend/FwLite/FwLiteProjectSync/DryRunMiniLcmApi.cs
#	backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs
#	backend/FwLite/LcmCrdt/LcmCrdtKernel.cs
#	backend/FwLite/MiniLcm/IMiniLcmWriteApi.cs
#	backend/FwLite/MiniLcm/InMemoryApi.cs
#	backend/FwLite/MiniLcm/SyncHelpers/EntrySync.cs

.vscode/settings.json

@@ -15,13 +15,18 @@
     "i18n-ally.sortKeys": true,
     "i18n-ally.keystyle": "nested",
     "i18n-ally.tabStyle": "tab",
-    "eslint.experimental.useFlatConfig": true,
+    "eslint.useFlatConfig": true,
     "eslint.workingDirectories": [
-        { "directory": "./frontend" }
+      "./frontend",
+      "./frontend/viewer"
     ],
     "sort-imports.on-save": true,
     "editor.detectIndentation": false,
     "eslint.validate": [
+      "javascript",
+      "typescript",
+      "html",
+      "json",
       "svelte"
     ],
     "yaml.schemas": {

backend/FwHeadless/CrdtSyncService.cs

@@ -0,0 +1,27 @@
+using LcmCrdt;
+using LcmCrdt.RemoteSync;
+using SIL.Harmony;
+
+namespace FwHeadless;
+
+public class CrdtSyncService(
+    CrdtHttpSyncService httpSyncService,
+    IHttpClientFactory httpClientFactory,
+    CurrentProjectService currentProjectService,
+    DataModel dataModel,
+    ILogger<CrdtSyncService> logger)
+{
+    public async Task Sync()
+    {
+        var lexboxRemoteServer = await httpSyncService.CreateProjectSyncable(
+            currentProjectService.ProjectData,
+            httpClientFactory.CreateClient(FwHeadlessKernel.LexboxHttpClientName)
+        );
+        var syncResults = await dataModel.SyncWith(lexboxRemoteServer);
+        if (!syncResults.IsSynced) throw new InvalidOperationException("Sync failed");
+        logger.LogInformation(
+            "Synced with Lexbox, Downloaded changes: {MissingFromLocal}, Uploaded changes: {MissingFromRemote}",
+            syncResults.MissingFromLocal.Length,
+            syncResults.MissingFromRemote.Length);
+    }
+}

backend/FwHeadless/FwHeadlessKernel.cs

@@ -1,11 +1,14 @@
 using FwDataMiniLcmBridge;
+using FwHeadless.Services;
 using FwLiteProjectSync;
 using LcmCrdt;
+using Microsoft.Extensions.Options;
 
 namespace FwHeadless;
 
 public static class FwHeadlessKernel
 {
+    public const string LexboxHttpClientName = "LexboxHttpClient";
     public static void AddFwHeadless(this IServiceCollection services)
     {
         services
@@ -16,9 +19,18 @@ public static void AddFwHeadless(this IServiceCollection services)
             .ValidateOnStart();
         services.AddScoped<SendReceiveService>();
         services.AddScoped<ProjectLookupService>();
+        services.AddScoped<LogSanitizerService>();
+        services.AddScoped<SafeLoggingProgress>();
         services
             .AddLcmCrdtClient()
             .AddFwDataBridge()
             .AddFwLiteProjectSync();
+        services.AddScoped<CrdtSyncService>();
+        services.AddTransient<HttpClientAuthHandler>();
+        services.AddHttpClient(LexboxHttpClientName,
+            (provider, client) =>
+            {
+                client.BaseAddress = new Uri(provider.GetRequiredService<IOptions<FwHeadlessConfig>>().Value.LexboxUrl);
+            }).AddHttpMessageHandler<HttpClientAuthHandler>();
     }
-};
+}

backend/FwHeadless/HttpClientAuthHandler.cs

@@ -0,0 +1,73 @@
+using System.Net;
+using LexCore;
+using LexCore.Auth;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Options;
+
+namespace FwHeadless;
+
+public class HttpClientAuthHandler(IOptions<FwHeadlessConfig> config, IMemoryCache cache, ILogger<HttpClientAuthHandler> logger) : DelegatingHandler
+{
+    protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        throw new NotSupportedException("use async apis");
+    }
+
+    protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+    {
+        var lexboxUrl = new Uri(config.Value.LexboxUrl);
+        if (request.RequestUri?.Authority != lexboxUrl.Authority)
+        {
+            return await base.SendAsync(request, cancellationToken);
+        }
+        try
+        {
+            await SetAuthHeader(request, cancellationToken, lexboxUrl);
+        }
+        catch (Exception e)
+        {
+            throw new InvalidOperationException("Unable to set auth header", e);
+        }
+        return await base.SendAsync(request, cancellationToken);
+    }
+
+    private async Task SetAuthHeader(HttpRequestMessage request, CancellationToken cancellationToken, Uri lexboxUrl)
+    {
+        var cookieContainer = new CookieContainer();
+        cookieContainer.Add(new Cookie(LexAuthConstants.AuthCookieName, await GetToken(cancellationToken), null, lexboxUrl.Authority));
+        request.Headers.Add("Cookie", cookieContainer.GetCookieHeader(lexboxUrl));
+    }
+
+    private async ValueTask<string> GetToken(CancellationToken cancellationToken)
+    {
+        try
+        {
+            return await cache.GetOrCreateAsync("LexboxAuthToken",
+                async entry =>
+                {
+                    if (InnerHandler is null) throw new InvalidOperationException("InnerHandler is null");
+                    logger.LogInformation("Getting auth token");
+                    var client = new HttpClient(InnerHandler);
+                    client.BaseAddress = new Uri(config.Value.LexboxUrl);
+                    var response = await client.PostAsJsonAsync("/api/login",
+                        new LoginRequest(config.Value.LexboxPassword, config.Value.LexboxUsername),
+                        cancellationToken);
+                    response.EnsureSuccessStatusCode();
+                    var cookies = response.Headers.GetValues("Set-Cookie");
+                    var cookieContainer = new CookieContainer();
+                    cookieContainer.SetCookies(response.RequestMessage!.RequestUri!, cookies.Single());
+                    var authCookie = cookieContainer.GetAllCookies()
+                        .FirstOrDefault(c => c.Name == LexAuthConstants.AuthCookieName);
+                    if (authCookie is null) throw new InvalidOperationException("Auth cookie not found");
+                    entry.SetValue(authCookie.Value);
+                    entry.AbsoluteExpiration = authCookie.Expires;
+                    logger.LogInformation("Got auth token: {AuthToken}", authCookie.Value);
+                    return authCookie.Value;
+                }) ?? throw new NullReferenceException("unable to get the login token");
+        }
+        catch (Exception e)
+        {
+            throw new InvalidOperationException("Unable to get auth token", e);
+        }
+    }
+}

backend/FwHeadless/Program.cs

@@ -1,7 +1,9 @@
 using FwHeadless;
 using FwDataMiniLcmBridge;
+using FwDataMiniLcmBridge.Api;
 using FwLiteProjectSync;
 using LcmCrdt;
+using LcmCrdt.RemoteSync;
 using LexData;
 using Microsoft.AspNetCore.Http.HttpResults;
 using Microsoft.Extensions.Options;
@@ -25,6 +27,14 @@
 
 var app = builder.Build();
 
+// Add lexbox-version header to all requests
+app.Logger.LogInformation("FwHeadless version: {version}", AppVersionService.Version);
+app.Use(async (context, next) =>
+{
+    context.Response.Headers["lexbox-version"] = AppVersionService.Version;
+    await next();
+});
+
 // Configure the HTTP request pipeline.
 if (app.Environment.IsDevelopment())
 {
@@ -37,11 +47,11 @@
 
 app.MapHealthChecks("/api/healthz");
 
-app.MapPost("/sync", ExecuteMergeRequest);
+app.MapPost("/api/crdt-sync", ExecuteMergeRequest);
 
 app.Run();
 
-static async Task<Results<Ok<CrdtFwdataProjectSyncService.SyncResult>, NotFound>> ExecuteMergeRequest(
+static async Task<Results<Ok<CrdtFwdataProjectSyncService.SyncResult>, NotFound, ProblemHttpResult>> ExecuteMergeRequest(
     ILogger<Program> logger,
     IServiceProvider services,
     SendReceiveService srService,
@@ -50,6 +60,8 @@
     ProjectsService projectsService,
     ProjectLookupService projectLookupService,
     CrdtFwdataProjectSyncService syncService,
+    CrdtHttpSyncService crdtHttpSyncService,
+    IHttpClientFactory httpClientFactory,
     Guid projectId,
     bool dryRun = false)
 {
@@ -67,6 +79,11 @@
         return TypedResults.NotFound();
     }
     logger.LogInformation("Project code is {projectCode}", projectCode);
+    //if we can't sync with lexbox fail fast
+    if (!await crdtHttpSyncService.TestAuth(httpClientFactory.CreateClient(FwHeadlessKernel.LexboxHttpClientName)))
+    {
+        return TypedResults.Problem("Unable to authenticate with Lexbox");
+    }
 
     var projectFolder = Path.Join(config.Value.ProjectStorageRoot, $"{projectCode}-{projectId}");
     if (!Directory.Exists(projectFolder)) Directory.CreateDirectory(projectFolder);
@@ -77,25 +94,71 @@
     logger.LogDebug("crdtFile: {crdtFile}", crdtFile);
     logger.LogDebug("fwDataFile: {fwDataFile}", fwDataProject.FilePath);
 
+    var fwdataApi = await SetupFwData(fwDataProject, srService, projectCode, logger, fwDataFactory);
+    using var deferCloseFwData = fwDataFactory.DeferClose(fwDataProject);
+    var crdtProject = await SetupCrdtProject(crdtFile, projectLookupService, projectId, projectsService, projectFolder, fwdataApi.ProjectId, config.Value.LexboxUrl);
+
+    var miniLcmApi = await services.OpenCrdtProject(crdtProject);
+    var crdtSyncService = services.GetRequiredService<CrdtSyncService>();
+    await crdtSyncService.Sync();
+
+
+    var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun);
+    logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges);
+
+    await crdtSyncService.Sync();
+    var srResult2 = await srService.SendReceive(fwDataProject, projectCode);
+    logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output);
+    return TypedResults.Ok(result);
+}
+
+static async Task<FwDataMiniLcmApi> SetupFwData(FwDataProject fwDataProject,
+    SendReceiveService srService,
+    string projectCode,
+    ILogger<Program> logger,
+    FwDataFactory fwDataFactory)
+{
     if (File.Exists(fwDataProject.FilePath))
     {
-        var srResult = srService.SendReceive(fwDataProject, projectCode);
+        var srResult = await srService.SendReceive(fwDataProject, projectCode);
         logger.LogInformation("Send/Receive result: {srResult}", srResult.Output);
     }
     else
     {
-        var srResult = srService.Clone(fwDataProject, projectCode);
+        var srResult = await srService.Clone(fwDataProject, projectCode);
         logger.LogInformation("Send/Receive result: {srResult}", srResult.Output);
     }
+
     var fwdataApi = fwDataFactory.GetFwDataMiniLcmApi(fwDataProject, true);
-    var crdtProject = File.Exists(crdtFile) ?
-        new CrdtProject("crdt", crdtFile) :
-        await projectsService.CreateProject(new("crdt", SeedNewProjectData: false, Path: projectFolder, FwProjectId: fwdataApi.ProjectId));
-    var miniLcmApi = await services.OpenCrdtProject(crdtProject);
-    var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun);
-    logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges);
-    var srResult2 = srService.SendReceive(fwDataProject, projectCode);
-    logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output);
-    return TypedResults.Ok(result);
+    return fwdataApi;
+}
+
+static async Task<CrdtProject> SetupCrdtProject(string crdtFile,
+    ProjectLookupService projectLookupService,
+    Guid projectId,
+    ProjectsService projectsService,
+    string projectFolder,
+    Guid fwProjectId,
+    string lexboxUrl)
+{
+    if (File.Exists(crdtFile))
+    {
+        return new CrdtProject("crdt", crdtFile);
+    }
+    else
+    {
+        if (await projectLookupService.IsCrdtProject(projectId))
+        {
+            //todo determine what to do in this case, maybe we just download the project?
+            throw new InvalidOperationException("Project already exists, not sure why it's not on the server");
+        }
+        return await projectsService.CreateProject(new("crdt",
+            SeedNewProjectData: false,
+            Id: projectId,
+            Path: projectFolder,
+            FwProjectId: fwProjectId,
+            Domain: new Uri(lexboxUrl)));
+    }
+
 }
 

backend/FwHeadless/ProjectLookupService.cs

@@ -1,5 +1,6 @@
 using LexData;
 using Microsoft.EntityFrameworkCore;
+using SIL.Harmony.Core;
 
 namespace FwHeadless;
 
@@ -13,4 +14,9 @@ public class ProjectLookupService(LexBoxDbContext dbContext)
             .FirstOrDefaultAsync();
         return projectCode;
     }
+
+    public async Task<bool> IsCrdtProject(Guid projectId)
+    {
+        return await dbContext.Set<ServerCommit>().AnyAsync(c => c.ProjectId == projectId);
+    }
 }

backend/FwHeadless/SendReceiveHelpers.cs

@@ -17,11 +17,15 @@ public SendReceiveAuth(FwHeadlessConfig config) : this(config.LexboxUsername, co
 
     public record LfMergeBridgeResult(string Output, string ProgressMessages);
 
-    private static LfMergeBridgeResult CallLfMergeBridge(string method, IDictionary<string, string> flexBridgeOptions)
+    private static async Task<LfMergeBridgeResult> CallLfMergeBridge(string method, IDictionary<string, string> flexBridgeOptions, IProgress? progress = null)
     {
-        var progress = new StringBuilderProgress();
-        LfMergeBridge.LfMergeBridge.Execute(method, progress, flexBridgeOptions.ToDictionary(), out var lfMergeBridgeOutputForClient);
-        return new LfMergeBridgeResult(lfMergeBridgeOutputForClient, progress.ToString());
+        var sbProgress = new StringBuilderProgress();
+        var lfMergeBridgeOutputForClient = await Task.Run(() =>
+        {
+            LfMergeBridge.LfMergeBridge.Execute(method, progress ?? sbProgress, flexBridgeOptions.ToDictionary(), out var output);
+            return output;
+        });
+        return new LfMergeBridgeResult(lfMergeBridgeOutputForClient, progress == null ? sbProgress.ToString() : "");
     }
 
     private static Uri BuildSendReceiveUrl(string baseUrl, string projectCode, SendReceiveAuth? auth)
@@ -45,7 +49,7 @@ private static Uri BuildSendReceiveUrl(string baseUrl, string projectCode, SendR
         return builder.Uri;
     }
 
-    public static LfMergeBridgeResult SendReceive(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", string? commitMessage = null)
+    public static async Task<LfMergeBridgeResult> SendReceive(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", string? commitMessage = null, IProgress? progress = null)
     {
         projectCode ??= project.Name;
         var fwdataInfo = new FileInfo(project.FilePath);
@@ -65,10 +69,10 @@ public static LfMergeBridgeResult SendReceive(FwDataProject project, string? pro
             { "user", "LexBox" },
         };
         if (commitMessage is not null) flexBridgeOptions["commitMessage"] = commitMessage;
-        return CallLfMergeBridge("Language_Forge_Send_Receive", flexBridgeOptions);
+        return await CallLfMergeBridge("Language_Forge_Send_Receive", flexBridgeOptions, progress);
     }
 
-    public static LfMergeBridgeResult CloneProject(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072")
+    public static async Task<LfMergeBridgeResult> CloneProject(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", IProgress? progress = null)
     {
         projectCode ??= project.Name;
         var fwdataInfo = new FileInfo(project.FilePath);
@@ -84,6 +88,6 @@ public static LfMergeBridgeResult CloneProject(FwDataProject project, string? pr
             { "languageDepotRepoUri", repoUrl.ToString() },
             { "deleteRepoIfNoSuchBranch", "false" },
         };
-        return CallLfMergeBridge("Language_Forge_Clone", flexBridgeOptions);
+        return await CallLfMergeBridge("Language_Forge_Clone", flexBridgeOptions, progress);
     }
 }