From e5fb026953506368dce7698111421b939971a030 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 18 Nov 2024 14:35:39 +0700 Subject: [PATCH 01/14] rework how AuthHelpers.cs get made using a LexboxServer instead of a Uri --- backend/FwLite/LocalWebApp/Auth/AuthConfig.cs | 8 +++ .../FwLite/LocalWebApp/Auth/AuthHelpers.cs | 21 ++++-- .../LocalWebApp/Auth/AuthHelpersFactory.cs | 29 ++++---- .../Services/LexboxProjectService.cs | 67 ++++++++++++------- 4 files changed, 81 insertions(+), 44 deletions(-) diff --git a/backend/FwLite/LocalWebApp/Auth/AuthConfig.cs b/backend/FwLite/LocalWebApp/Auth/AuthConfig.cs index 4e11f8706..542aae2db 100644 --- a/backend/FwLite/LocalWebApp/Auth/AuthConfig.cs +++ b/backend/FwLite/LocalWebApp/Auth/AuthConfig.cs @@ -1,4 +1,5 @@ using System.ComponentModel.DataAnnotations; +using LcmCrdt; namespace LocalWebApp.Auth; @@ -15,6 +16,13 @@ public LexboxServer GetServerByAuthority(string authority) { return LexboxServers.FirstOrDefault(s => s.Authority.Authority == authority) ?? throw new ArgumentException($"Server {authority} not found"); } + + public LexboxServer GetServer(ProjectData projectData) + { + var originDomain = projectData.OriginDomain; + if (string.IsNullOrEmpty(originDomain)) throw new InvalidOperationException("No origin domain in project data"); + return GetServerByAuthority(new Uri(originDomain).Authority); + } public LexboxServer GetServer(string serverName) { return LexboxServers.FirstOrDefault(s => s.DisplayName == serverName) ?? throw new ArgumentException($"Server {serverName} not found"); diff --git a/backend/FwLite/LocalWebApp/Auth/AuthHelpers.cs b/backend/FwLite/LocalWebApp/Auth/AuthHelpers.cs index 14022c72f..429cac81a 100644 --- a/backend/FwLite/LocalWebApp/Auth/AuthHelpers.cs +++ b/backend/FwLite/LocalWebApp/Auth/AuthHelpers.cs @@ -1,6 +1,7 @@ using System.Net.Http.Headers; using System.Security.Cryptography; using LocalWebApp.Routes; +using LocalWebApp.Services; using Microsoft.Extensions.Options; using Microsoft.Identity.Client; using Microsoft.Identity.Client.Extensions.Msal; @@ -21,7 +22,8 @@ public class AuthHelpers private readonly IHttpMessageHandlerFactory _httpMessageHandlerFactory; private readonly OAuthService _oAuthService; private readonly UrlContext _urlContext; - private readonly Uri _authority; + private readonly LexboxServer _lexboxServer; + private readonly LexboxProjectService _lexboxProjectService; private readonly ILogger _logger; private readonly IPublicClientApplication _application; AuthenticationResult? _authResult; @@ -32,14 +34,16 @@ public AuthHelpers(LoggerAdapter loggerAdapter, LinkGenerator linkGenerator, OAuthService oAuthService, UrlContext urlContext, - Uri authority, + LexboxServer lexboxServer, + LexboxProjectService lexboxProjectService, ILogger logger, IHostEnvironment hostEnvironment) { _httpMessageHandlerFactory = httpMessageHandlerFactory; _oAuthService = oAuthService; _urlContext = urlContext; - _authority = authority; + _lexboxServer = lexboxServer; + _lexboxProjectService = lexboxProjectService; _logger = logger; (var hostUrl, _isRedirectHostGuess) = urlContext.GetUrl(); _redirectHost = HostString.FromUriComponent(hostUrl); @@ -56,7 +60,7 @@ public AuthHelpers(LoggerAdapter loggerAdapter, .WithLogging(loggerAdapter, hostEnvironment.IsDevelopment()) .WithHttpClientFactory(new HttpClientFactoryAdapter(httpMessageHandlerFactory)) .WithRedirectUri(redirectUri) - .WithOidcAuthority(authority.ToString()) + .WithOidcAuthority(lexboxServer.Authority.ToString()) .Build(); _ = MsalCacheHelper.CreateAsync(BuildCacheProperties(options.Value.CacheFileName)).ContinueWith( task => @@ -111,6 +115,7 @@ public HttpClient GetHttpClient() public async Task SignIn(CancellationToken cancellation = default) { + InvalidateProjectCache(); return await _oAuthService.SubmitLoginRequest(_application, cancellation); } @@ -122,6 +127,12 @@ public async Task Logout() { await _application.RemoveAsync(account); } + InvalidateProjectCache(); + } + + private void InvalidateProjectCache() + { + _lexboxProjectService.InvalidateProjectsCache(_lexboxServer); } private async ValueTask GetAuth() @@ -177,7 +188,7 @@ await _application var handler = _httpMessageHandlerFactory.CreateHandler(AuthHttpClientName); var client = new HttpClient(handler, false); - client.BaseAddress = _authority; + client.BaseAddress = _lexboxServer.Authority; client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", auth.AccessToken); return client; } diff --git a/backend/FwLite/LocalWebApp/Auth/AuthHelpersFactory.cs b/backend/FwLite/LocalWebApp/Auth/AuthHelpersFactory.cs index bc5b2489b..801e3b6d8 100644 --- a/backend/FwLite/LocalWebApp/Auth/AuthHelpersFactory.cs +++ b/backend/FwLite/LocalWebApp/Auth/AuthHelpersFactory.cs @@ -1,33 +1,34 @@ using System.Collections.Concurrent; using LcmCrdt; +using LocalWebApp.Services; +using Microsoft.Extensions.Options; namespace LocalWebApp.Auth; public class AuthHelpersFactory( IServiceProvider provider, ProjectContext projectContext, + IOptions options, IHttpContextAccessor contextAccessor) { private readonly ConcurrentDictionary _helpers = new(); - private string AuthorityKey(Uri authority) => - authority.GetComponents(UriComponents.HostAndPort, UriFormat.Unescaped); + private string AuthorityKey(LexboxServer server) => "AuthHelper|" + server.Authority.Authority; /// - /// gets an Auth Helper for the given authority + /// gets an Auth Helper for the given server /// - /// should include scheme, host and port, no path - public AuthHelpers GetHelper(Uri authority) + public AuthHelpers GetHelper(LexboxServer server) { - var helper = _helpers.GetOrAdd(AuthorityKey(authority), - static (host, arg) => ActivatorUtilities.CreateInstance(arg.provider, arg.authority), - (authority, provider)); + var helper = _helpers.GetOrAdd(AuthorityKey(server), + static (host, arg) => ActivatorUtilities.CreateInstance(arg.provider, arg.server), + (server, provider)); //an auth helper can get created based on the server host, however in development that will not be the same as the client host //so we need to recreate it if the host is not valid if (!helper.IsHostUrlValid()) { - _helpers.TryRemove(AuthorityKey(authority), out _); - return GetHelper(authority); + _helpers.TryRemove(AuthorityKey(server), out _); + return GetHelper(server); } return helper; @@ -38,14 +39,10 @@ public AuthHelpers GetHelper(Uri authority) /// public AuthHelpers GetHelper(ProjectData project) { + ; var originDomain = project.OriginDomain; if (string.IsNullOrEmpty(originDomain)) throw new InvalidOperationException("No origin domain in project data"); - return GetHelper(new Uri(originDomain)); - } - - public AuthHelpers GetHelper(LexboxServer server) - { - return GetHelper(server.Authority); + return GetHelper(options.Value.GetServer(project)); } /// diff --git a/backend/FwLite/LocalWebApp/Services/LexboxProjectService.cs b/backend/FwLite/LocalWebApp/Services/LexboxProjectService.cs index 6cc650852..2dc5af1c4 100644 --- a/backend/FwLite/LocalWebApp/Services/LexboxProjectService.cs +++ b/backend/FwLite/LocalWebApp/Services/LexboxProjectService.cs @@ -12,23 +12,39 @@ public class LexboxProjectService( ILogger logger, IHttpMessageHandlerFactory httpMessageHandlerFactory, BackgroundSyncService backgroundSyncService, + IOptions options, IMemoryCache cache) { - public record LexboxCrdtProject(Guid Id, string Name); + public record LexboxProject(Guid Id, string Code, string Name, bool IsFwDataProject, bool IsCrdtProject); - public async Task GetLexboxProjects(LexboxServer server) + public LexboxServer[] Servers() { - var httpClient = await helpersFactory.GetHelper(server).CreateClient(); - if (httpClient is null) return []; - try - { - return await httpClient.GetFromJsonAsync("api/crdt/listProjects") ?? []; - } - catch (HttpRequestException e) - { - logger.LogError(e, "Error getting lexbox projects"); - return []; - } + return options.Value.LexboxServers; + } + + public async Task GetLexboxProjects(LexboxServer server) + { + return await cache.GetOrCreateAsync(CacheKey(server), + async entry => + { + entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(5); + var httpClient = await helpersFactory.GetHelper(server).CreateClient(); + if (httpClient is null) return []; + try + { + return await httpClient.GetFromJsonAsync("api/crdt/listProjects") ?? []; + } + catch (HttpRequestException e) + { + logger.LogError(e, "Error getting lexbox projects"); + return []; + } + }) ?? []; + } + + private static string CacheKey(LexboxServer server) + { + return $"Projects|{server.Authority.Authority}"; } public async Task GetLexboxProjectId(LexboxServer server, string code) @@ -46,29 +62,34 @@ public async Task GetLexboxProjects(LexboxServer server) } } + public void InvalidateProjectsCache(LexboxServer server) + { + cache.Remove(CacheKey(server)); + } + public async Task ListenForProjectChanges(ProjectData projectData, CancellationToken stoppingToken) { if (string.IsNullOrEmpty(projectData.OriginDomain)) return; - var lexboxConnection = await StartLexboxProjectChangeListener(projectData.OriginDomain, stoppingToken); + var lexboxConnection = await StartLexboxProjectChangeListener(options.Value.GetServer(projectData), stoppingToken); if (lexboxConnection is null) return; await lexboxConnection.SendAsync("ListenForProjectChanges", projectData.Id, stoppingToken); } - private string CacheKey(string originDomain) => $"LexboxProjectChangeListener|{originDomain}"; + private static string HubConnectionCacheKey(LexboxServer server) => $"LexboxProjectChangeListener|{server.Authority.Authority}"; - public async Task StartLexboxProjectChangeListener(string originDomain, + public async Task StartLexboxProjectChangeListener(LexboxServer server, CancellationToken stoppingToken) { HubConnection? connection; - if (cache.TryGetValue(CacheKey(originDomain), out connection) && connection is not null) + if (cache.TryGetValue(HubConnectionCacheKey(server), out connection) && connection is not null) { return connection; } - if (await helpersFactory.GetHelper(new Uri(originDomain)).GetCurrentToken() is null) + if (await helpersFactory.GetHelper(server).GetCurrentToken() is null) { - logger.LogWarning("Unable to create signalR client, user is not authenticated to {OriginDomain}", originDomain); + logger.LogWarning("Unable to create signalR client, user is not authenticated to {OriginDomain}", server.Authority); return null; } @@ -76,7 +97,7 @@ public async Task ListenForProjectChanges(ProjectData projectData, CancellationT //todo bridge logging to the aspnet logger .ConfigureLogging(logging => logging.AddConsole()) .WithAutomaticReconnect() - .WithUrl($"{originDomain}/api/hub/crdt/project-changes", + .WithUrl($"{server.Authority}/api/hub/crdt/project-changes", connectionOptions => { connectionOptions.HttpMessageHandlerFactory = handler => @@ -85,7 +106,7 @@ public async Task ListenForProjectChanges(ProjectData projectData, CancellationT return httpMessageHandlerFactory.CreateHandler(AuthHelpers.AuthHttpClientName); }; connectionOptions.AccessTokenProvider = - async () => await helpersFactory.GetHelper(new Uri(originDomain)).GetCurrentToken(); + async () => await helpersFactory.GetHelper(server).GetCurrentToken(); }) .Build(); @@ -105,10 +126,10 @@ public async Task ListenForProjectChanges(ProjectData projectData, CancellationT connection.Closed += async (exception) => { - cache.Remove(CacheKey(originDomain)); + cache.Remove(HubConnectionCacheKey(server)); await connection.DisposeAsync(); }; - cache.CreateEntry(CacheKey(originDomain)).SetValue(connection).RegisterPostEvictionCallback( + cache.CreateEntry(HubConnectionCacheKey(server)).SetValue(connection).RegisterPostEvictionCallback( static (key, value, reason, state) => { if (value is HubConnection con) From e79afab0cfce1688cb5b401545cb98d957fd7e29 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 18 Nov 2024 16:05:12 +0700 Subject: [PATCH 02/14] fix ProjectData caching issue --- backend/FwLite/LcmCrdt/CurrentProjectService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/FwLite/LcmCrdt/CurrentProjectService.cs b/backend/FwLite/LcmCrdt/CurrentProjectService.cs index f2c69dd05..336ad465c 100644 --- a/backend/FwLite/LcmCrdt/CurrentProjectService.cs +++ b/backend/FwLite/LcmCrdt/CurrentProjectService.cs @@ -27,7 +27,7 @@ public async ValueTask GetProjectData() private static string CacheKey(CrdtProject project) { - return project.DbPath + "|ProjectData"; + return project.Name + "|ProjectData"; } private static string CacheKey(Guid projectId) From efc73b4f0013628cd64c879b17b946685de0c88a Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 18 Nov 2024 16:06:31 +0700 Subject: [PATCH 03/14] move user projects query into ProjectService.cs --- .../LexBoxApi/Controllers/CrdtController.cs | 21 ++++++++++++------- backend/LexBoxApi/GraphQL/LexQueries.cs | 5 +++-- backend/LexBoxApi/Services/ProjectService.cs | 7 +++++++ 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/backend/LexBoxApi/Controllers/CrdtController.cs b/backend/LexBoxApi/Controllers/CrdtController.cs index c982eb99f..4c4e98bb0 100644 --- a/backend/LexBoxApi/Controllers/CrdtController.cs +++ b/backend/LexBoxApi/Controllers/CrdtController.cs @@ -1,8 +1,9 @@ using System.Text.Json.Serialization; +using LexBoxApi.Auth; using SIL.Harmony.Core; -using LexBoxApi.Auth.Attributes; using LexBoxApi.Hub; using LexBoxApi.Services; +using LexCore.Entities; using LexCore.ServiceInterfaces; using LexData; using Microsoft.AspNetCore.Mvc; @@ -19,13 +20,16 @@ public class CrdtController( LexBoxDbContext dbContext, IHubContext hubContext, IPermissionService permissionService, + LoggedInContext loggedInContext, ProjectService projectService) : ControllerBase { + private DbSet ServerCommits => dbContext.Set(); + [HttpGet("{projectId}/get")] public async Task> GetSyncState(Guid projectId) { await permissionService.AssertCanSyncProject(projectId); - return await dbContext.Set().Where(c => c.ProjectId == projectId).GetSyncState(); + return await ServerCommits.Where(c => c.ProjectId == projectId).GetSyncState(); } [HttpPost("{projectId}/add")] @@ -49,24 +53,25 @@ public record ChangesResult(IAsyncEnumerable MissingFromClient, Sy [JsonIgnore]//just to ensure type safety IEnumerable IChangesResult.MissingFromClient => MissingFromClient.ToBlockingEnumerable(); } + [HttpPost("{projectId}/changes")] public async Task> Changes(Guid projectId, [FromBody] SyncState clientHeads) { await permissionService.AssertCanSyncProject(projectId); - var commits = dbContext.Set().Where(c => c.ProjectId == projectId); + var commits = ServerCommits.Where(c => c.ProjectId == projectId); var localState = await commits.GetSyncState(); return new ChangesResult(commits.GetMissingCommits(localState, clientHeads), localState); } - public record LexboxCrdtProject(Guid Id, string Name); + public record FwLiteProject(Guid Id, string Code, string Name, bool IsFwDataProject, bool IsCrdtProject); [HttpGet("listProjects")] - public async Task> ListProjects() + public async Task> ListProjects() { - return await dbContext.Projects - .Where(p => dbContext.Set().Any(c => c.ProjectId == p.Id)) - .Select(p => new LexboxCrdtProject(p.Id, p.Code)) + return await projectService.UserProjects(loggedInContext.User.Id) + .Where(p => p.Type == ProjectType.FLEx) + .Select(p => new FwLiteProject(p.Id, p.Code, p.Name, p.LastCommit != null, ServerCommits.Any(c => c.ProjectId == p.Id))) .ToArrayAsync(); } diff --git a/backend/LexBoxApi/GraphQL/LexQueries.cs b/backend/LexBoxApi/GraphQL/LexQueries.cs index 3a3f0b10d..edef6b2de 100644 --- a/backend/LexBoxApi/GraphQL/LexQueries.cs +++ b/backend/LexBoxApi/GraphQL/LexQueries.cs @@ -2,6 +2,7 @@ using LexBoxApi.Auth; using LexBoxApi.Auth.Attributes; using LexBoxApi.GraphQL.CustomTypes; +using LexBoxApi.Services; using LexCore.Auth; using LexCore.Entities; using LexCore.ServiceInterfaces; @@ -19,11 +20,11 @@ public class LexQueries public async Task> MyProjects( LexAuthService lexAuthService, LoggedInContext loggedInContext, - LexBoxDbContext dbContext, + ProjectService projectService, IResolverContext context) { var userId = loggedInContext.User.Id; - var myProjects = await dbContext.Projects.Where(p => p.Users.Select(u => u.UserId).Contains(userId)) + var myProjects = await projectService.UserProjects(userId) .AsNoTracking().Project(context).ToListAsync(); if (loggedInContext.User.IsOutOfSyncWithMyProjects(myProjects)) diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index 298f9faff..095f43582 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -1,6 +1,7 @@ using System.Data.Common; using LexBoxApi.Models.Project; using LexBoxApi.Services.Email; +using LexCore.Auth; using LexCore.Config; using LexCore.Entities; using LexCore.Exceptions; @@ -38,6 +39,7 @@ public async Task CreateProject(CreateProjectInput input) IsConfidential = isConfidentialIsUntrustworthy ? null : input.IsConfidential, Organizations = theOrg is not null ? [theOrg] : [], Users = input.ProjectManagerId.HasValue ? [new() { UserId = input.ProjectManagerId.Value, Role = ProjectRole.Manager }] : [], + FlexProjectMetadata = input.Type == ProjectType.FLEx ? new() : null }); // Also delete draft project, if any await dbContext.DraftProjects.Where(dp => dp.Id == projectId).ExecuteDeleteAsync(); @@ -302,4 +304,9 @@ public async Task ResetLexEntryCount(string projectCode) } return count; } + + public IQueryable UserProjects(Guid userId) + { + return dbContext.Projects.Where(p => p.Users.Select(u => u.UserId).Contains(userId)); + } } From c7e3f5913c1f1d27acc85efb6f71342ed6c76707 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 18 Nov 2024 16:07:09 +0700 Subject: [PATCH 04/14] allow user to choose which project to upload to from FW Lite --- backend/FwLite/LocalWebApp/LocalAppKernel.cs | 2 +- .../LocalWebApp/Routes/ProjectRoutes.cs | 16 +++++------ frontend/viewer/src/lib/SyncConfig.svelte | 27 +++++++++++++++++-- .../src/lib/services/projects-service.ts | 4 +-- 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/backend/FwLite/LocalWebApp/LocalAppKernel.cs b/backend/FwLite/LocalWebApp/LocalAppKernel.cs index 2ad02e7ca..aa27e512d 100644 --- a/backend/FwLite/LocalWebApp/LocalAppKernel.cs +++ b/backend/FwLite/LocalWebApp/LocalAppKernel.cs @@ -21,7 +21,7 @@ public static IServiceCollection AddLocalAppServices(this IServiceCollection ser services.AddAuthHelpers(environment); services.AddSingleton(); services.AddScoped(); - services.AddScoped(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs b/backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs index 1d945394c..f275b10eb 100644 --- a/backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs +++ b/backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs @@ -4,6 +4,7 @@ using LocalWebApp.Auth; using LocalWebApp.Hubs; using LocalWebApp.Services; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using MiniLcm; using MiniLcm.Models; @@ -24,7 +25,9 @@ public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication ap foreach (var server in options.Value.LexboxServers) { var lexboxProjects = await lexboxProjectService.GetLexboxProjects(server); - serversProjects.Add(server.Authority.Authority, lexboxProjects.Select(p => new ProjectModel(p.Name, false, false, true, server.Authority.Authority, p.Id)).ToArray()); + serversProjects.Add(server.Authority.Authority, lexboxProjects.Select(p => new ProjectModel + (p.Name, Crdt: p.IsCrdtProject, Fwdata: false, Lexbox: true, server.Authority.Authority, p.Id)) + .ToArray()); } return serversProjects; @@ -77,16 +80,13 @@ public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication ap SyncService syncService, IOptions options, CurrentProjectService currentProjectService, - string serverAuthority) => + string serverAuthority, + [FromQuery] Guid lexboxProjectId) => { var server = options.Value.GetServerByAuthority(serverAuthority); - var foundProjectGuid = - await lexboxProjectService.GetLexboxProjectId(server, currentProjectService.ProjectData.Name); - if (foundProjectGuid is null) - return Results.BadRequest( - $"Project code {currentProjectService.ProjectData.Name} not found on lexbox"); - await currentProjectService.SetProjectSyncOrigin(server.Authority, foundProjectGuid); + await currentProjectService.SetProjectSyncOrigin(server.Authority, lexboxProjectId); await syncService.ExecuteSync(); + lexboxProjectService.InvalidateProjectsCache(server); return TypedResults.Ok(); }); group.MapPost("/download/crdt/{serverAuthority}/{newProjectName}", diff --git a/frontend/viewer/src/lib/SyncConfig.svelte b/frontend/viewer/src/lib/SyncConfig.svelte index c038879be..f018081ea 100644 --- a/frontend/viewer/src/lib/SyncConfig.svelte +++ b/frontend/viewer/src/lib/SyncConfig.svelte @@ -15,6 +15,7 @@ throw error; }); }); + let isUploaded = false; let projectServer = writable(null, set => { projectsService.getProjectServer(projectName).then(server => { @@ -36,11 +37,18 @@ }; let uploading = false; + let targetProjectId: string | null = null; + async function serverProjectsForUpload(serverAuthority: string) { + const remoteProjects = await projectsService.fetchRemoteProjects(); + return remoteProjects[serverAuthority].filter(p => !p.crdt); + } + async function upload() { if (!$projectServer) return; + if (!targetProjectId) return; uploading = true; //todo if not logged in then login - await projectsService.uploadCrdtProject($projectServer, projectName); + await projectsService.uploadCrdtProject($projectServer, projectName, targetProjectId); uploading = false; isUploaded = true; } @@ -65,7 +73,22 @@ {/if} {#if $projectServer && !isUploaded && server.loggedIn} - {:else if $projectServer && !isUploaded && !server.loggedIn} diff --git a/frontend/viewer/src/lib/services/projects-service.ts b/frontend/viewer/src/lib/services/projects-service.ts index 32090a542..76f26153d 100644 --- a/frontend/viewer/src/lib/services/projects-service.ts +++ b/frontend/viewer/src/lib/services/projects-service.ts @@ -42,8 +42,8 @@ export class ProjectService { } } - async uploadCrdtProject(server: string, projectName: string) { - await fetch(`/api/upload/crdt/${server}/${projectName}`, {method: 'POST'}); + async uploadCrdtProject(server: string, projectName: string, lexboxProjectId: string) { + await fetch(`/api/upload/crdt/${server}/${projectName}?lexboxProjectId=${lexboxProjectId}`, {method: 'POST'}); } async getProjectServer(projectName: string): Promise { const projects = await this.fetchProjects(); From a8959ba475027c0a4e8365afd0e4a7fcc9a5b5c4 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 18 Nov 2024 16:15:08 +0700 Subject: [PATCH 05/14] simplify matching of local and remote projects --- frontend/viewer/src/HomeView.svelte | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frontend/viewer/src/HomeView.svelte b/frontend/viewer/src/HomeView.svelte index 90e60321c..3e0159498 100644 --- a/frontend/viewer/src/HomeView.svelte +++ b/frontend/viewer/src/HomeView.svelte @@ -104,10 +104,6 @@ if (project.id) { matches = projects.find(p => p.id == project.id && p.serverAuthority == project.serverAuthority); } - //for now the local project list does not include the id, so fallback to the name - if (!matches) { - matches = projects.find(p => p.name === project.name && p.serverAuthority == project.serverAuthority); - } return matches; } From 4e164b3ac3a5665d095bb610afd03d79777f8cf0 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 18 Nov 2024 16:33:03 +0700 Subject: [PATCH 06/14] show a message about being required to login again whe your login has expired for a project that's already syncing --- backend/FwLite/LocalWebApp/Auth/AuthHelpers.cs | 4 ++-- backend/FwLite/LocalWebApp/Auth/OAuthService.cs | 16 +++++++++++----- backend/FwLite/LocalWebApp/Routes/AuthRoutes.cs | 13 ++++++++----- frontend/viewer/src/lib/SyncConfig.svelte | 9 ++++++--- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/backend/FwLite/LocalWebApp/Auth/AuthHelpers.cs b/backend/FwLite/LocalWebApp/Auth/AuthHelpers.cs index 429cac81a..b33738f06 100644 --- a/backend/FwLite/LocalWebApp/Auth/AuthHelpers.cs +++ b/backend/FwLite/LocalWebApp/Auth/AuthHelpers.cs @@ -113,10 +113,10 @@ public HttpClient GetHttpClient() } } - public async Task SignIn(CancellationToken cancellation = default) + public async Task SignIn(string returnUrl, CancellationToken cancellation = default) { InvalidateProjectCache(); - return await _oAuthService.SubmitLoginRequest(_application, cancellation); + return await _oAuthService.SubmitLoginRequest(_application, returnUrl, cancellation); } public async Task Logout() diff --git a/backend/FwLite/LocalWebApp/Auth/OAuthService.cs b/backend/FwLite/LocalWebApp/Auth/OAuthService.cs index 60069a70f..fe9d9c94a 100644 --- a/backend/FwLite/LocalWebApp/Auth/OAuthService.cs +++ b/backend/FwLite/LocalWebApp/Auth/OAuthService.cs @@ -12,14 +12,16 @@ namespace LocalWebApp.Auth; public class OAuthService(ILogger logger, IHostApplicationLifetime applicationLifetime, IOptions options) : BackgroundService { public record SignInResult(Uri? AuthUri, bool HandledBySystemWebView); - public async Task SubmitLoginRequest(IPublicClientApplication application, CancellationToken cancellation) + public async Task SubmitLoginRequest(IPublicClientApplication application, + string returnUrl, + CancellationToken cancellation) { if (options.Value.SystemWebViewLogin) { await HandleSystemWebViewLogin(application, cancellation); return new(null, true); } - var request = new OAuthLoginRequest(application); + var request = new OAuthLoginRequest(application, returnUrl); if (!_requestChannel.Writer.TryWrite(request)) { throw new InvalidOperationException("Only one request at a time"); @@ -40,7 +42,7 @@ private async Task HandleSystemWebViewLogin(IPublicClientApplication application .ExecuteAsync(cancellation); } - public async Task FinishLoginRequest(Uri uri, CancellationToken cancellation = default) + public async Task<(AuthenticationResult, string ClientReturnUrl)> FinishLoginRequest(Uri uri, CancellationToken cancellation = default) { var queryString = HttpUtility.ParseQueryString(uri.Query); var state = queryString.Get("state") ?? throw new InvalidOperationException("State is null"); @@ -48,7 +50,7 @@ public async Task FinishLoginRequest(Uri uri, Cancellation throw new InvalidOperationException("Invalid state"); //step 5 request.SetReturnUri(uri); - return await request.GetAuthenticationResult(applicationLifetime.ApplicationStopping.Merge(cancellation)); + return (await request.GetAuthenticationResult(applicationLifetime.ApplicationStopping.Merge(cancellation)), request.ClientReturnUrl); //step 8 } @@ -90,7 +92,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) /// instead we have to do this so we can use the currently open browser, redirect it to the auth url passed in here and then once it's done and the callback comes to our server, /// send that call to here so that MSAL can pull out the access token /// -public class OAuthLoginRequest(IPublicClientApplication app) : ICustomWebUi +public class OAuthLoginRequest(IPublicClientApplication app, string clientReturnUrl) : ICustomWebUi { public IPublicClientApplication Application { get; } = app; public string? State { get; private set; } @@ -124,4 +126,8 @@ public void SetException(Exception e) } public Task GetAuthenticationResult(CancellationToken cancellation) => _resultTcs.Task.WaitAsync(cancellation); + /// + /// url to return the client to once the login is finished + /// + public string ClientReturnUrl { get; } = clientReturnUrl; } diff --git a/backend/FwLite/LocalWebApp/Routes/AuthRoutes.cs b/backend/FwLite/LocalWebApp/Routes/AuthRoutes.cs index 2cd52d1e8..2d8950bb0 100644 --- a/backend/FwLite/LocalWebApp/Routes/AuthRoutes.cs +++ b/backend/FwLite/LocalWebApp/Routes/AuthRoutes.cs @@ -1,6 +1,7 @@ using System.Security.AccessControl; using System.Web; using LocalWebApp.Auth; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; namespace LocalWebApp.Routes; @@ -23,12 +24,14 @@ public static IEndpointConventionBuilder MapAuthRoutes(this WebApplication app) }); }); group.MapGet("/login/{authority}", - async (AuthHelpersFactory factory, string authority, IOptions options) => + async (AuthHelpersFactory factory, string authority, IOptions options, [FromHeader] string referer, ILogger logger) => { - var result = await factory.GetHelper(options.Value.GetServerByAuthority(authority)).SignIn(); + var returnUrl = new Uri(referer).PathAndQuery; + logger.LogInformation("Login redirect to {ReturnUrl}", returnUrl); + var result = await factory.GetHelper(options.Value.GetServerByAuthority(authority)).SignIn(returnUrl); if (result.HandledBySystemWebView) { - return Results.Redirect("/"); + return Results.Redirect(returnUrl); } if (result.AuthUri is null) throw new InvalidOperationException("AuthUri is null"); @@ -43,8 +46,8 @@ public static IEndpointConventionBuilder MapAuthRoutes(this WebApplication app) context.Request.Path); uriBuilder.Query = context.Request.QueryString.ToUriComponent(); - await oAuthService.FinishLoginRequest(uriBuilder.Uri); - return Results.Redirect("/"); + var (_, returnUrl) = await oAuthService.FinishLoginRequest(uriBuilder.Uri); + return Results.Redirect(returnUrl); }).WithName(CallbackRoute); group.MapGet("/me/{authority}", async (AuthHelpersFactory factory, string authority, IOptions options) => diff --git a/frontend/viewer/src/lib/SyncConfig.svelte b/frontend/viewer/src/lib/SyncConfig.svelte index f018081ea..1b694a7a5 100644 --- a/frontend/viewer/src/lib/SyncConfig.svelte +++ b/frontend/viewer/src/lib/SyncConfig.svelte @@ -67,7 +67,7 @@ fieldActions={(elem) => /* a hack to disable typing/filtering */ {elem.readOnly = true; return [];}} search={() => /* a hack to always show all options */ Promise.resolve()}> -{:else if isUploaded} +{:else if isUploaded && server.loggedIn} @@ -91,7 +91,10 @@ -{:else if $projectServer && !isUploaded && !server.loggedIn} - +{/if} +{#if server && !server.loggedIn} + {#if isUploaded} + Your login has expired to sync with {server.displayName}. Please login again. + {/if} {/if} From 8d7b79f6034f8c94b2125c7757bd57d0ed52e444 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 18 Nov 2024 16:38:35 +0700 Subject: [PATCH 07/14] only show crdt projects from remote servers on the home page --- frontend/viewer/src/HomeView.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/viewer/src/HomeView.svelte b/frontend/viewer/src/HomeView.svelte index 3e0159498..04b3d45b1 100644 --- a/frontend/viewer/src/HomeView.svelte +++ b/frontend/viewer/src/HomeView.svelte @@ -231,7 +231,7 @@ {/if} - {@const serverProjects = remoteProjects[server.authority] ?? []} + {@const serverProjects = remoteProjects[server.authority]?.filter(p => p.crdt) ?? []} {#each serverProjects as project} {@const localProject = matchesProject(projects, project)}
From 0f11643ddfb4321a60bd2192e646035fa58b3d0d Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Mon, 18 Nov 2024 17:02:03 +0700 Subject: [PATCH 08/14] remove logging --- backend/FwLite/LocalWebApp/Routes/AuthRoutes.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/FwLite/LocalWebApp/Routes/AuthRoutes.cs b/backend/FwLite/LocalWebApp/Routes/AuthRoutes.cs index 2d8950bb0..21f521741 100644 --- a/backend/FwLite/LocalWebApp/Routes/AuthRoutes.cs +++ b/backend/FwLite/LocalWebApp/Routes/AuthRoutes.cs @@ -24,10 +24,9 @@ public static IEndpointConventionBuilder MapAuthRoutes(this WebApplication app) }); }); group.MapGet("/login/{authority}", - async (AuthHelpersFactory factory, string authority, IOptions options, [FromHeader] string referer, ILogger logger) => + async (AuthHelpersFactory factory, string authority, IOptions options, [FromHeader] string referer) => { var returnUrl = new Uri(referer).PathAndQuery; - logger.LogInformation("Login redirect to {ReturnUrl}", returnUrl); var result = await factory.GetHelper(options.Value.GetServerByAuthority(authority)).SignIn(returnUrl); if (result.HandledBySystemWebView) { From d5da247ab3d1db84c8172502e10b98e8396da477 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 19 Nov 2024 11:27:54 +0700 Subject: [PATCH 09/14] always show crdt column --- frontend/viewer/src/HomeView.svelte | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/frontend/viewer/src/HomeView.svelte b/frontend/viewer/src/HomeView.svelte index 04b3d45b1..527240b71 100644 --- a/frontend/viewer/src/HomeView.svelte +++ b/frontend/viewer/src/HomeView.svelte @@ -81,14 +81,10 @@ name: 'fwdata', header: 'FieldWorks', }, - ...($isDev - ? [ - { - name: 'crdt', - header: 'CRDT', - }, - ] - : []), + { + name: 'crdt', + header: 'CRDT', + }, ...(servers.find(s => s.loggedIn) ? [ { From 4a4e72d8932c5aeed811c7b62a599ef17b6d74ba Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 19 Nov 2024 16:11:04 +0700 Subject: [PATCH 10/14] set project OriginDomain to null if the initial sync fails --- backend/FwLite/LcmCrdt/CurrentProjectService.cs | 2 +- backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/backend/FwLite/LcmCrdt/CurrentProjectService.cs b/backend/FwLite/LcmCrdt/CurrentProjectService.cs index 336ad465c..e99edde78 100644 --- a/backend/FwLite/LcmCrdt/CurrentProjectService.cs +++ b/backend/FwLite/LcmCrdt/CurrentProjectService.cs @@ -57,7 +57,7 @@ private void RemoveProjectDataCache() memoryCache.Remove(CacheKey(Project)); } - public async Task SetProjectSyncOrigin(Uri domain, Guid? id) + public async Task SetProjectSyncOrigin(Uri? domain, Guid? id) { var originDomain = ProjectData.GetOriginDomain(domain); if (id is null) diff --git a/backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs b/backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs index f275b10eb..6afe5caba 100644 --- a/backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs +++ b/backend/FwLite/LocalWebApp/Routes/ProjectRoutes.cs @@ -85,7 +85,15 @@ public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication ap { var server = options.Value.GetServerByAuthority(serverAuthority); await currentProjectService.SetProjectSyncOrigin(server.Authority, lexboxProjectId); - await syncService.ExecuteSync(); + try + { + await syncService.ExecuteSync(); + } + catch + { + await currentProjectService.SetProjectSyncOrigin(null, null); + throw; + } lexboxProjectService.InvalidateProjectsCache(server); return TypedResults.Ok(); }); From 0c117d6fd1ff480e680dc2600a4ce73c54430dc9 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 19 Nov 2024 10:53:38 +0100 Subject: [PATCH 11/14] Prevent simultaneous imports This is already partially assumed in the code. --- frontend/viewer/src/HomeView.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/viewer/src/HomeView.svelte b/frontend/viewer/src/HomeView.svelte index 527240b71..0d6b2069c 100644 --- a/frontend/viewer/src/HomeView.svelte +++ b/frontend/viewer/src/HomeView.svelte @@ -1,4 +1,4 @@ - diff --git a/frontend/viewer/src/lib/services/projects-service.ts b/frontend/viewer/src/lib/services/projects-service.ts index 76f26153d..cb273ce1b 100644 --- a/frontend/viewer/src/lib/services/projects-service.ts +++ b/frontend/viewer/src/lib/services/projects-service.ts @@ -28,23 +28,35 @@ export class ProjectService { return {error: undefined}; } - async importFwDataProject(name: string) { - await fetch(`/api/import/fwdata/${name}`, { + async importFwDataProject(name: string): Promise { + const r = await fetch(`/api/import/fwdata/${name}`, { method: 'POST', }); + if (!r.ok) { + AppNotification.display(`Failed to import FieldWorks project ${name}: ${r.statusText} (${r.status})`, 'error', 'long'); + console.error(`Failed to import FieldWorks project ${name}: ${r.statusText} (${r.status})`, r, await r.text()) + } + return r.ok; } async downloadCrdtProject(project: Project) { const r = await fetch(`/api/download/crdt/${project.serverAuthority}/${project.name}`, {method: 'POST'}); - if (r.status !== 200) { - AppNotification.display(`Failed to download project, status code ${r.status}`, 'error'); - console.error(`Failed to download project ${project.name}`, r) + if (!r.ok) { + AppNotification.display(`Failed to download project ${project.name}: ${r.statusText} (${r.status})`, 'error', 'long'); + console.error(`Failed to download project ${project.name}: ${r.statusText} (${r.status})`, r, await r.text()) } + return r.ok; } - async uploadCrdtProject(server: string, projectName: string, lexboxProjectId: string) { - await fetch(`/api/upload/crdt/${server}/${projectName}?lexboxProjectId=${lexboxProjectId}`, {method: 'POST'}); + async uploadCrdtProject(server: string, projectName: string, lexboxProjectId: string): Promise { + const r = await fetch(`/api/upload/crdt/${server}/${projectName}?lexboxProjectId=${lexboxProjectId}`, {method: 'POST'}); + if (!r.ok) { + AppNotification.display(`Failed to upload project ${projectName}: ${r.statusText} (${r.status})`, 'error', 'long'); + console.error(`Failed to upload project ${projectName}: ${r.statusText} (${r.status})`, r, await r.text()) + } + return r.ok; } + async getProjectServer(projectName: string): Promise { const projects = await this.fetchProjects(); //todo project server is always null from local projects` From 7464218701becf59880a60b45531f68bf355ddb7 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Tue, 19 Nov 2024 10:59:31 +0100 Subject: [PATCH 13/14] UI tweaks --- frontend/viewer/src/HomeView.svelte | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/frontend/viewer/src/HomeView.svelte b/frontend/viewer/src/HomeView.svelte index 0d6b2069c..199e2f65a 100644 --- a/frontend/viewer/src/HomeView.svelte +++ b/frontend/viewer/src/HomeView.svelte @@ -1,10 +1,12 @@ -