From 9abc2a395d4d32fa9c55e5cfc2c0ee437b493953 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 19 Nov 2024 12:17:49 +0700 Subject: [PATCH 1/7] add crdt sync passthrough endpoint to call crdt sync on fw headless, setup service discovery to simplify configuration --- backend/FwHeadless/Program.cs | 5 +++-- .../CrdtFwdataProjectSyncService.cs | 3 +-- backend/LexBoxApi/Controllers/CrdtController.cs | 11 ++++++++++- backend/LexBoxApi/LexBoxApi.csproj | 1 + backend/LexBoxApi/LexBoxKernel.cs | 3 +++ backend/LexBoxApi/Services/FwHeadlessClient.cs | 13 +++++++++++++ backend/LexBoxApi/appsettings.Development.json | 7 +++++++ backend/LexBoxApi/appsettings.json | 5 +++++ backend/LexCore/Sync/SyncResult.cs | 3 +++ deployment/base/lexbox-deployment.yaml | 3 +++ 10 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 backend/LexBoxApi/Services/FwHeadlessClient.cs create mode 100644 backend/LexCore/Sync/SyncResult.cs diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index e4e897016..95e175463 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -4,6 +4,7 @@ using FwLiteProjectSync; using LcmCrdt; using LcmCrdt.RemoteSync; +using LexCore.Sync; using LexData; using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Extensions.Options; @@ -51,7 +52,7 @@ app.Run(); -static async Task, NotFound, ProblemHttpResult>> ExecuteMergeRequest( +static async Task, NotFound, ProblemHttpResult>> ExecuteMergeRequest( ILogger logger, IServiceProvider services, SendReceiveService srService, @@ -69,7 +70,7 @@ if (dryRun) { logger.LogInformation("Dry run, not actually syncing"); - return TypedResults.Ok(new CrdtFwdataProjectSyncService.SyncResult(0, 0)); + return TypedResults.Ok(new SyncResult(0, 0)); } var projectCode = await projectLookupService.GetProjectCode(projectId); diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index 12258dcee..ee42e1026 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -1,6 +1,7 @@ using System.Text.Json; using FwDataMiniLcmBridge.Api; using LcmCrdt; +using LexCore.Sync; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MiniLcm; @@ -13,8 +14,6 @@ namespace FwLiteProjectSync; public class CrdtFwdataProjectSyncService(IOptions lcmCrdtConfig, MiniLcmImport miniLcmImport, ILogger logger) { - public record SyncResult(int CrdtChanges, int FwdataChanges); - public async Task Sync(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataApi, bool dryRun = false) { if (crdtApi is CrdtMiniLcmApi crdt && crdt.ProjectData.FwProjectId != fwdataApi.ProjectId) diff --git a/backend/LexBoxApi/Controllers/CrdtController.cs b/backend/LexBoxApi/Controllers/CrdtController.cs index c982eb99f..5baba55ef 100644 --- a/backend/LexBoxApi/Controllers/CrdtController.cs +++ b/backend/LexBoxApi/Controllers/CrdtController.cs @@ -4,6 +4,7 @@ using LexBoxApi.Hub; using LexBoxApi.Services; using LexCore.ServiceInterfaces; +using LexCore.Sync; using LexData; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; @@ -19,7 +20,8 @@ public class CrdtController( LexBoxDbContext dbContext, IHubContext hubContext, IPermissionService permissionService, - ProjectService projectService) : ControllerBase + ProjectService projectService, + FwHeadlessClient fwHeadlessClient) : ControllerBase { [HttpGet("{projectId}/get")] public async Task> GetSyncState(Guid projectId) @@ -85,4 +87,11 @@ public async Task> GetProjectId(string code) return Ok(projectId); } + + [HttpPost("crdt-sync/{projectId}")] + [AdminRequired] + public async Task ExecuteMerge(Guid projectId) + { + return await fwHeadlessClient.CrdtSync(projectId); + } } diff --git a/backend/LexBoxApi/LexBoxApi.csproj b/backend/LexBoxApi/LexBoxApi.csproj index baffc9fb9..238fc7e57 100644 --- a/backend/LexBoxApi/LexBoxApi.csproj +++ b/backend/LexBoxApi/LexBoxApi.csproj @@ -33,6 +33,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs index 347449122..a92217cf2 100644 --- a/backend/LexBoxApi/LexBoxKernel.cs +++ b/backend/LexBoxApi/LexBoxKernel.cs @@ -49,6 +49,9 @@ public static void AddLexBoxApi(this IServiceCollection services, .ValidateDataAnnotations() .ValidateOnStart(); services.AddHttpClient(); + services.AddServiceDiscovery(); + services.AddHttpClient(client => client.BaseAddress = new ("http://fwHeadless")) + .AddServiceDiscovery();//service discovery means that we lookup the hostname in Services__fwHeadless__http in config services.AddHttpContextAccessor(); services.AddMemoryCache(); services.AddScoped(); diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs new file mode 100644 index 000000000..8a7e921d3 --- /dev/null +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -0,0 +1,13 @@ +using LexCore.Sync; + +namespace LexBoxApi.Services; + +public class FwHeadlessClient(HttpClient httpClient) +{ + public async Task CrdtSync(Guid projectId) + { + var response = await httpClient.PostAsync($"/api/crdt-sync?projectId={projectId}", null); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync(); + } +} diff --git a/backend/LexBoxApi/appsettings.Development.json b/backend/LexBoxApi/appsettings.Development.json index 35f3ab4dc..1705db6f7 100644 --- a/backend/LexBoxApi/appsettings.Development.json +++ b/backend/LexBoxApi/appsettings.Development.json @@ -77,5 +77,12 @@ "From": "Lexbox ", "EmailRenderHost": "localhost:3000", "BaseUrl": "http://localhost:3000" + }, + "Services": { + "fwHeadless": { + "http": [ + "localhost:5275" + ] + } } } diff --git a/backend/LexBoxApi/appsettings.json b/backend/LexBoxApi/appsettings.json index 6f3e5ef2e..3ae6b1f17 100644 --- a/backend/LexBoxApi/appsettings.json +++ b/backend/LexBoxApi/appsettings.json @@ -70,5 +70,10 @@ }, "Email": { "CreateProjectEmailDestination": "lexbox_support@groups.sil.org" + }, + "Services": { + "fwHeadless": { + "http": ["fw-headless"] + } } } diff --git a/backend/LexCore/Sync/SyncResult.cs b/backend/LexCore/Sync/SyncResult.cs new file mode 100644 index 000000000..70b7d0aa6 --- /dev/null +++ b/backend/LexCore/Sync/SyncResult.cs @@ -0,0 +1,3 @@ +namespace LexCore.Sync; + +public record SyncResult(int CrdtChanges, int FwdataChanges); diff --git a/deployment/base/lexbox-deployment.yaml b/deployment/base/lexbox-deployment.yaml index 15f25a185..a7c4da81a 100644 --- a/deployment/base/lexbox-deployment.yaml +++ b/deployment/base/lexbox-deployment.yaml @@ -205,6 +205,9 @@ spec: value: /tmp/tus-test-upload - name: Tus__ResetUploadPath value: /tmp/tus-reset-upload + - name: Services__fwHeadless__http__0 + value: fw-headless + - name: otel-collector image: otel/opentelemetry-collector-contrib:0.101.0 From 12f192adefa69734936e6a6bb2ccf27407ba33e7 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 19 Nov 2024 13:19:28 +0700 Subject: [PATCH 2/7] fix issue with cookie port number not working when in the host parameter --- Tiltfile | 2 +- backend/FwHeadless/HttpClientAuthHandler.cs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Tiltfile b/Tiltfile index 24b320e75..061bdb63d 100644 --- a/Tiltfile +++ b/Tiltfile @@ -28,7 +28,7 @@ docker_build( context='backend', dockerfile='./backend/FwHeadless/dev.Dockerfile', only=['.'], - ignore=['LexBoxApi'], + ignore=['LexBoxApi', '**/Mercurial', '**/MercurialExtensions'], live_update=[ sync('backend', '/src/backend') ] diff --git a/backend/FwHeadless/HttpClientAuthHandler.cs b/backend/FwHeadless/HttpClientAuthHandler.cs index cc3efa799..41d364334 100644 --- a/backend/FwHeadless/HttpClientAuthHandler.cs +++ b/backend/FwHeadless/HttpClientAuthHandler.cs @@ -34,7 +34,10 @@ protected override async Task SendAsync(HttpRequestMessage private async Task SetAuthHeader(HttpRequestMessage request, CancellationToken cancellationToken, Uri lexboxUrl) { var cookieContainer = new CookieContainer(); - cookieContainer.Add(new Cookie(LexAuthConstants.AuthCookieName, await GetToken(cancellationToken), null, lexboxUrl.Authority)); + cookieContainer.Add(new Cookie(LexAuthConstants.AuthCookieName, await GetToken(cancellationToken), null, lexboxUrl.Host) + { + Port = $"\"{lexboxUrl.Port}\"" + }); request.Headers.Add("Cookie", cookieContainer.GetCookieHeader(lexboxUrl)); } From fd68f0396df8b9619f0e6cc2edee10668903670e Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 19 Nov 2024 13:24:06 +0700 Subject: [PATCH 3/7] log errors from fw headless, and return a problem from the lexbox api --- backend/LexBoxApi/Controllers/CrdtController.cs | 6 ++++-- backend/LexBoxApi/Services/FwHeadlessClient.cs | 12 +++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/backend/LexBoxApi/Controllers/CrdtController.cs b/backend/LexBoxApi/Controllers/CrdtController.cs index 5baba55ef..d340ae458 100644 --- a/backend/LexBoxApi/Controllers/CrdtController.cs +++ b/backend/LexBoxApi/Controllers/CrdtController.cs @@ -90,8 +90,10 @@ public async Task> GetProjectId(string code) [HttpPost("crdt-sync/{projectId}")] [AdminRequired] - public async Task ExecuteMerge(Guid projectId) + public async Task> ExecuteMerge(Guid projectId) { - return await fwHeadlessClient.CrdtSync(projectId); + var result = await fwHeadlessClient.CrdtSync(projectId); + if (result is null) return Problem("Failed to sync CRDT"); + return result; } } diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 8a7e921d3..4366ecf00 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -2,12 +2,18 @@ namespace LexBoxApi.Services; -public class FwHeadlessClient(HttpClient httpClient) +public class FwHeadlessClient(HttpClient httpClient, ILogger logger) { public async Task CrdtSync(Guid projectId) { var response = await httpClient.PostAsync($"/api/crdt-sync?projectId={projectId}", null); - response.EnsureSuccessStatusCode(); - return await response.Content.ReadFromJsonAsync(); + if (response.IsSuccessStatusCode) + return await response.Content.ReadFromJsonAsync(); + logger.LogError("Failed to sync CRDT: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + response.StatusCode, + response.StatusCode, + projectId, + await response.Content.ReadAsStringAsync()); + return null; } } From 7f476a6a685972b397a4811720c263402e8d3138 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 19 Nov 2024 14:37:35 +0100 Subject: [PATCH 4/7] Fix duplicate error log parameter --- backend/LexBoxApi/Services/FwHeadlessClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 4366ecf00..48324f218 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -11,7 +11,7 @@ public class FwHeadlessClient(HttpClient httpClient, ILogger l return await response.Content.ReadFromJsonAsync(); logger.LogError("Failed to sync CRDT: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", response.StatusCode, - response.StatusCode, + response.ReasonPhrase, projectId, await response.Content.ReadAsStringAsync()); return null; From 0d923c317104e69f1bb1f2b66f52958a99ede787 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 19 Nov 2024 15:46:02 +0100 Subject: [PATCH 5/7] Fix Elawa project is not seeded with FwData property --- backend/LexData/SeedingData.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/LexData/SeedingData.cs b/backend/LexData/SeedingData.cs index bb7e2fb44..553699e8b 100644 --- a/backend/LexData/SeedingData.cs +++ b/backend/LexData/SeedingData.cs @@ -156,6 +156,7 @@ private async Task SeedUserData(CancellationToken cancellationToken = default) Description = "Eastern Lawa project", Code = "elawa-dev-flex", Type = ProjectType.FLEx, + FlexProjectMetadata = new(), ProjectOrigin = ProjectMigrationStatus.Migrated, LastCommit = DateTimeOffset.UtcNow, RetentionPolicy = RetentionPolicy.Dev, From ba52653ca105ee6c9e7000fbaa83599f540b2912 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 19 Nov 2024 15:56:19 +0100 Subject: [PATCH 6/7] Add button for triggering CRDT sync --- frontend/src/lib/forms/Button.svelte | 7 ++- frontend/src/lib/icons/Icon.svelte | 4 +- .../project/[project_code]/+page.svelte | 2 + .../[project_code]/CrdtSyncButton.svelte | 43 +++++++++++++++++++ 4 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte diff --git a/frontend/src/lib/forms/Button.svelte b/frontend/src/lib/forms/Button.svelte index b2f8c1e9c..595e951ba 100644 --- a/frontend/src/lib/forms/Button.svelte +++ b/frontend/src/lib/forms/Button.svelte @@ -7,13 +7,16 @@ export let type: undefined | 'submit' = undefined; export let size: undefined | 'btn-sm' = undefined; export let disabled = false; + export let customLoader = false; diff --git a/frontend/src/lib/icons/Icon.svelte b/frontend/src/lib/icons/Icon.svelte index 99fa52775..874b49d13 100644 --- a/frontend/src/lib/icons/Icon.svelte +++ b/frontend/src/lib/icons/Icon.svelte @@ -10,6 +10,8 @@ export let size: IconSize = 'text-lg'; export let color: `text-${string}` | undefined = undefined; export let pale = false; + export let spin = false; + export let spinReverse = false; // For pixel perfect text alignment, because the svgs often contain vertical white-space export let y: string | undefined = undefined; @@ -17,5 +19,5 @@ {#if icon} - + {/if} diff --git a/frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte b/frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte index ec4a4bb70..76ea3ef35 100644 --- a/frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte +++ b/frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte @@ -55,6 +55,7 @@ import { onMount } from 'svelte'; import { getSearchParamValues } from '$lib/util/query-params'; import FlexModelVersionText from '$lib/components/Projects/FlexModelVersionText.svelte'; + import CrdtSyncButton from './CrdtSyncButton.svelte'; export let data: PageData; $: user = data.user; @@ -312,6 +313,7 @@ {/if} {#if project.type === ProjectType.FlEx && $isDev} + {:else} diff --git a/frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte b/frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte new file mode 100644 index 000000000..4c27935b3 --- /dev/null +++ b/frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte @@ -0,0 +1,43 @@ + + + From 2d636512dc3c61669d490ed5bbca221fec654044 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 21 Nov 2024 10:02:11 +0700 Subject: [PATCH 7/7] increase request timeout on the sync endpoint, change the path to not be redundant --- backend/LexBoxApi/Controllers/CrdtController.cs | 4 +++- .../project/[project_code]/CrdtSyncButton.svelte | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/LexBoxApi/Controllers/CrdtController.cs b/backend/LexBoxApi/Controllers/CrdtController.cs index d340ae458..6a601377d 100644 --- a/backend/LexBoxApi/Controllers/CrdtController.cs +++ b/backend/LexBoxApi/Controllers/CrdtController.cs @@ -6,6 +6,7 @@ using LexCore.ServiceInterfaces; using LexCore.Sync; using LexData; +using Microsoft.AspNetCore.Http.Timeouts; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; @@ -88,8 +89,9 @@ public async Task> GetProjectId(string code) return Ok(projectId); } - [HttpPost("crdt-sync/{projectId}")] + [HttpPost("sync/{projectId}")] [AdminRequired] + [RequestTimeout(300_000)]//5 minutes public async Task> ExecuteMerge(Guid projectId) { var result = await fwHeadlessClient.CrdtSync(projectId); diff --git a/frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte b/frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte index 4c27935b3..324c40126 100644 --- a/frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte +++ b/frontend/src/routes/(authenticated)/project/[project_code]/CrdtSyncButton.svelte @@ -12,7 +12,7 @@ async function triggerSync(): Promise { syncing = true; try { - const response = await fetch(`/api/crdt/crdt-sync/${projectId}`, { + const response = await fetch(`/api/crdt/sync/${projectId}`, { method: 'POST', });