From baf6cd5145c2a186df32f592dce8ea51101526ef Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Mon, 28 Oct 2024 14:59:46 +0700 Subject: [PATCH] Use project ID, not code, in CrdtMerge API (#1169) * Revisit filenames we use for CrdtMerge Use project code and ID in root folder name, use "crdt" and "fw" inside root folder no matter what the project's actual code or name is. * Update CRDT caching so same filename isn't a problem Using the name "crdt" everywhere as a project name caused a couple of caching issues in existing CRDT code, but the change is pretty easy. * Now use project ID, not code, in CrdtMerge API Old URL: /sync?projectCode=sena-3 New URL: /sync?projectId=(guid) --------- Co-authored-by: Kevin Hahn --- backend/CrdtMerge/CrdtMerge.csproj | 1 + backend/CrdtMerge/CrdtMergeKernel.cs | 1 + backend/CrdtMerge/Program.cs | 46 ++++++++++++------- backend/CrdtMerge/ProjectLookupService.cs | 16 +++++++ backend/CrdtMerge/SendReceiveHelpers.cs | 12 ++--- backend/CrdtMerge/SendReceiveService.cs | 6 ++- .../CrdtMerge/appsettings.Development.json | 3 ++ .../FwDataMiniLcmBridge/FwDataFactory.cs | 2 +- .../CrdtFwdataProjectSyncService.cs | 1 + .../FwLite/LcmCrdt.Tests/LcmCrdt.Tests.csproj | 2 +- .../FwLite/LcmCrdt.Tests/OpenProjectTests.cs | 26 +++++++++++ .../FwLite/LcmCrdt/CurrentProjectService.cs | 2 +- backend/FwLite/LcmCrdt/LcmCrdtKernel.cs | 10 +++- backend/LexData/DataKernel.cs | 3 +- 14 files changed, 101 insertions(+), 30 deletions(-) create mode 100644 backend/CrdtMerge/ProjectLookupService.cs create mode 100644 backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs diff --git a/backend/CrdtMerge/CrdtMerge.csproj b/backend/CrdtMerge/CrdtMerge.csproj index d828a99e4..0380eb189 100644 --- a/backend/CrdtMerge/CrdtMerge.csproj +++ b/backend/CrdtMerge/CrdtMerge.csproj @@ -20,6 +20,7 @@ + diff --git a/backend/CrdtMerge/CrdtMergeKernel.cs b/backend/CrdtMerge/CrdtMergeKernel.cs index caa7e6113..0c312c074 100644 --- a/backend/CrdtMerge/CrdtMergeKernel.cs +++ b/backend/CrdtMerge/CrdtMergeKernel.cs @@ -15,6 +15,7 @@ public static void AddCrdtMerge(this IServiceCollection services) .ValidateDataAnnotations() .ValidateOnStart(); services.AddScoped(); + services.AddScoped(); services .AddLcmCrdtClient() .AddFwDataBridge() diff --git a/backend/CrdtMerge/Program.cs b/backend/CrdtMerge/Program.cs index 030692c48..e2604a9ec 100644 --- a/backend/CrdtMerge/Program.cs +++ b/backend/CrdtMerge/Program.cs @@ -2,6 +2,8 @@ using FwDataMiniLcmBridge; using FwLiteProjectSync; using LcmCrdt; +using LexData; +using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Extensions.Options; using MiniLcm; using Scalar.AspNetCore; @@ -12,6 +14,11 @@ // Learn more about configuring OpenAPI at https://aka.ms/aspnet/openapi builder.Services.AddOpenApi(); +builder.Services.AddLexData( + autoApplyMigrations: false, + useOpenIddict: false +); + builder.Services.AddCrdtMerge(); var app = builder.Build(); @@ -30,56 +37,61 @@ app.Run(); -static async Task ExecuteMergeRequest( +static async Task, NotFound>> ExecuteMergeRequest( ILogger logger, IServiceProvider services, SendReceiveService srService, IOptions config, FwDataFactory fwDataFactory, ProjectsService projectsService, + ProjectLookupService projectLookupService, CrdtFwdataProjectSyncService syncService, - string projectCode, - // string projectName, // TODO: Add this to the API eventually + Guid projectId, bool dryRun = false) { - logger.LogInformation("About to execute sync request for {projectCode}", projectCode); + logger.LogInformation("About to execute sync request for {projectId}", projectId); if (dryRun) { logger.LogInformation("Dry run, not actually syncing"); - return new(0, 0); + return TypedResults.Ok(new CrdtFwdataProjectSyncService.SyncResult(0, 0)); + } + + var projectCode = await projectLookupService.GetProjectCode(projectId); + if (projectCode is null) + { + logger.LogError("Project ID {projectId} not found", projectId); + return TypedResults.NotFound(); } + logger.LogInformation("Project code is {projectCode}", projectCode); - // TODO: Instead of projectCode here, we'll evetually look up project ID and use $"{projectName}-{projectId}" as the project folder - var projectFolder = Path.Join(config.Value.ProjectStorageRoot, projectCode); + var projectFolder = Path.Join(config.Value.ProjectStorageRoot, $"{projectCode}-{projectId}"); if (!Directory.Exists(projectFolder)) Directory.CreateDirectory(projectFolder); - // TODO: add projectName parameter and use it instead of projectCode here - var crdtFile = Path.Join(projectFolder, $"{projectCode}.sqlite"); + var crdtFile = Path.Join(projectFolder, "crdt.sqlite"); - var fwDataProject = new FwDataProject(projectCode, projectFolder); // TODO: use projectName (once we have it) instead of projectCode here + var fwDataProject = new FwDataProject("fw", projectFolder); logger.LogDebug("crdtFile: {crdtFile}", crdtFile); logger.LogDebug("fwDataFile: {fwDataFile}", fwDataProject.FilePath); if (File.Exists(fwDataProject.FilePath)) { - var srResult = srService.SendReceive(fwDataProject); + var srResult = srService.SendReceive(fwDataProject, projectCode); logger.LogInformation("Send/Receive result: {srResult}", srResult.Output); } else { - var srResult = srService.Clone(fwDataProject); + var srResult = srService.Clone(fwDataProject, projectCode); logger.LogInformation("Send/Receive result: {srResult}", srResult.Output); } var fwdataApi = fwDataFactory.GetFwDataMiniLcmApi(fwDataProject, true); - // var crdtProject = projectsService.GetProject(crdtProjectName); var crdtProject = File.Exists(crdtFile) ? - new CrdtProject(projectCode, crdtFile) : // TODO: use projectName (once we have it) instead of projectCode here - await projectsService.CreateProject(new(projectCode, SeedNewProjectData: false, Path: projectFolder, FwProjectId: fwdataApi.ProjectId)); + new CrdtProject("crdt", crdtFile) : + await projectsService.CreateProject(new("crdt", SeedNewProjectData: false, Path: projectFolder, FwProjectId: fwdataApi.ProjectId)); var miniLcmApi = await services.OpenCrdtProject(crdtProject); var result = await syncService.Sync(miniLcmApi, fwdataApi, dryRun); logger.LogInformation("Sync result, CrdtChanges: {CrdtChanges}, FwdataChanges: {FwdataChanges}", result.CrdtChanges, result.FwdataChanges); - var srResult2 = srService.SendReceive(fwDataProject); + var srResult2 = srService.SendReceive(fwDataProject, projectCode); logger.LogInformation("Send/Receive result after CRDT sync: {srResult2}", srResult2.Output); - return result; + return TypedResults.Ok(result); } diff --git a/backend/CrdtMerge/ProjectLookupService.cs b/backend/CrdtMerge/ProjectLookupService.cs new file mode 100644 index 000000000..ac8244e99 --- /dev/null +++ b/backend/CrdtMerge/ProjectLookupService.cs @@ -0,0 +1,16 @@ +using LexData; +using Microsoft.EntityFrameworkCore; + +namespace CrdtMerge; + +public class ProjectLookupService(LexBoxDbContext dbContext) +{ + public async ValueTask GetProjectCode(Guid projectId) + { + var projectCode = await dbContext.Projects + .Where(p => p.Id == projectId) + .Select(p => p.Code) + .FirstOrDefaultAsync(); + return projectCode; + } +} diff --git a/backend/CrdtMerge/SendReceiveHelpers.cs b/backend/CrdtMerge/SendReceiveHelpers.cs index b3fb283af..8eb1404c0 100644 --- a/backend/CrdtMerge/SendReceiveHelpers.cs +++ b/backend/CrdtMerge/SendReceiveHelpers.cs @@ -45,14 +45,14 @@ private static Uri BuildSendReceiveUrl(string baseUrl, string projectCode, SendR return builder.Uri; } - public static LfMergeBridgeResult SendReceive(FwDataProject project, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", string? commitMessage = null) + public static LfMergeBridgeResult SendReceive(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072", string? commitMessage = null) { - // If projectCode not given, calculate it from the fwdataPath + projectCode ??= project.Name; var fwdataInfo = new FileInfo(project.FilePath); if (fwdataInfo.Directory is null) throw new InvalidOperationException( $"Not allowed to Send/Receive root-level directories like C:\\, was '{project.FilePath}'"); - var repoUrl = BuildSendReceiveUrl(baseUrl, project.Name, auth); + var repoUrl = BuildSendReceiveUrl(baseUrl, projectCode, auth); var flexBridgeOptions = new Dictionary { @@ -68,13 +68,13 @@ public static LfMergeBridgeResult SendReceive(FwDataProject project, string base return CallLfMergeBridge("Language_Forge_Send_Receive", flexBridgeOptions); } - public static LfMergeBridgeResult CloneProject(FwDataProject project, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072") + public static LfMergeBridgeResult CloneProject(FwDataProject project, string? projectCode = null, string baseUrl = "http://localhost", SendReceiveAuth? auth = null, string fdoDataModelVersion = "7000072") { - // If projectCode not given, calculate it from the fwdataPath + projectCode ??= project.Name; var fwdataInfo = new FileInfo(project.FilePath); if (fwdataInfo.Directory is null) throw new InvalidOperationException($"Not allowed to Send/Receive root-level directories like C:\\ '{project.FilePath}'"); - var repoUrl = BuildSendReceiveUrl(baseUrl, project.Name, auth); + var repoUrl = BuildSendReceiveUrl(baseUrl, projectCode, auth); var flexBridgeOptions = new Dictionary { diff --git a/backend/CrdtMerge/SendReceiveService.cs b/backend/CrdtMerge/SendReceiveService.cs index 093427bec..14db577ae 100644 --- a/backend/CrdtMerge/SendReceiveService.cs +++ b/backend/CrdtMerge/SendReceiveService.cs @@ -5,10 +5,11 @@ namespace CrdtMerge; public class SendReceiveService(IOptions config) { - public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project, string? commitMessage = null) + public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project, string? projectCode, string? commitMessage = null) { return SendReceiveHelpers.SendReceive( project: project, + projectCode: projectCode, baseUrl: config.Value.HgWebUrl, auth: new SendReceiveHelpers.SendReceiveAuth(config.Value), fdoDataModelVersion: config.Value.FdoDataModelVersion, @@ -16,10 +17,11 @@ public SendReceiveHelpers.LfMergeBridgeResult SendReceive(FwDataProject project, ); } - public SendReceiveHelpers.LfMergeBridgeResult Clone(FwDataProject project) + public SendReceiveHelpers.LfMergeBridgeResult Clone(FwDataProject project, string? projectCode) { return SendReceiveHelpers.CloneProject( project: project, + projectCode: projectCode, baseUrl: config.Value.HgWebUrl, auth: new SendReceiveHelpers.SendReceiveAuth(config.Value), fdoDataModelVersion: config.Value.FdoDataModelVersion diff --git a/backend/CrdtMerge/appsettings.Development.json b/backend/CrdtMerge/appsettings.Development.json index c1e390265..0995c2046 100644 --- a/backend/CrdtMerge/appsettings.Development.json +++ b/backend/CrdtMerge/appsettings.Development.json @@ -6,6 +6,9 @@ "LexboxPassword": "pass", "FdoDataModelVersion": "7000072" }, + "DbConfig": { + "LexBoxConnectionString": "Host=localhost;Port=5433;Username=postgres;Password=972b722e63f549938d07bd8c4ee5086c;Database=lexbox;Include Error Detail=true" + }, "Logging": { "LogLevel": { "Default": "Information", diff --git a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs index 6322aff39..3490145a8 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs @@ -38,7 +38,7 @@ public FwDataMiniLcmApi GetFwDataMiniLcmApi(string projectName, bool saveOnDispo return GetFwDataMiniLcmApi(project, saveOnDispose); } - private string CacheKey(FwDataProject project) => $"{nameof(FwDataFactory)}|{project.FileName}"; + private string CacheKey(FwDataProject project) => $"{nameof(FwDataFactory)}|{project.FilePath}"; public FwDataMiniLcmApi GetFwDataMiniLcmApi(FwDataProject project, bool saveOnDispose) { diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index 8b7278591..5b18b044b 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -22,6 +22,7 @@ public async Task Sync(IMiniLcmApi crdtApi, FwDataMiniLcmApi fwdataA } var projectSnapshot = await GetProjectSnapshot(fwdataApi.Project.Name, fwdataApi.Project.ProjectsPath); SyncResult result = await Sync(crdtApi, fwdataApi, dryRun, fwdataApi.EntryCount, projectSnapshot); + fwdataApi.Save(); if (!dryRun) { diff --git a/backend/FwLite/LcmCrdt.Tests/LcmCrdt.Tests.csproj b/backend/FwLite/LcmCrdt.Tests/LcmCrdt.Tests.csproj index 1eacc4fa7..2b9260c62 100644 --- a/backend/FwLite/LcmCrdt.Tests/LcmCrdt.Tests.csproj +++ b/backend/FwLite/LcmCrdt.Tests/LcmCrdt.Tests.csproj @@ -14,7 +14,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs b/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs new file mode 100644 index 000000000..e53ea52e8 --- /dev/null +++ b/backend/FwLite/LcmCrdt.Tests/OpenProjectTests.cs @@ -0,0 +1,26 @@ +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace LcmCrdt.Tests; + +public class OpenProjectTests +{ + [Fact] + public async Task OpeningAProjectWorks() + { + var sqliteConnectionString = "OpeningAProjectWorks.sqlite"; + var builder = Host.CreateEmptyApplicationBuilder(null); + builder.Services.AddLcmCrdtClient(); + using var host = builder.Build(); + var services = host.Services; + var asyncScope = services.CreateAsyncScope(); + await asyncScope.ServiceProvider.GetRequiredService() + .CreateProject(new(Name: "OpeningAProjectWorks", Path: "")); + + var miniLcmApi = (CrdtMiniLcmApi)await asyncScope.ServiceProvider.OpenCrdtProject(new CrdtProject("OpeningAProjectWorks", sqliteConnectionString)); + miniLcmApi.ProjectData.Name.Should().Be("OpeningAProjectWorks"); + + await asyncScope.ServiceProvider.GetRequiredService().Database.EnsureDeletedAsync(); + } +} diff --git a/backend/FwLite/LcmCrdt/CurrentProjectService.cs b/backend/FwLite/LcmCrdt/CurrentProjectService.cs index 336ad465c..f2c69dd05 100644 --- a/backend/FwLite/LcmCrdt/CurrentProjectService.cs +++ b/backend/FwLite/LcmCrdt/CurrentProjectService.cs @@ -27,7 +27,7 @@ public async ValueTask GetProjectData() private static string CacheKey(CrdtProject project) { - return project.Name + "|ProjectData"; + return project.DbPath + "|ProjectData"; } private static string CacheKey(Guid projectId) diff --git a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs index eabf4aae5..93ea9587d 100644 --- a/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs +++ b/backend/FwLite/LcmCrdt/LcmCrdtKernel.cs @@ -152,10 +152,18 @@ public static void ConfigureCrdt(CrdtConfig config) .Add(); } - public static async Task OpenCrdtProject(this IServiceProvider services, CrdtProject project) + public static Task OpenCrdtProject(this IServiceProvider services, CrdtProject project) { + //this method must not be async, otherwise Setting the project scope will not work as expected. + //the project is stored in the async scope, if a new scope is created in this method then it will be gone once the method returns + //making the lcm api unusable var projectsService = services.GetRequiredService(); projectsService.SetProjectScope(project); + return LoadMiniLcmApi(services); + } + + private static async Task LoadMiniLcmApi(IServiceProvider services) + { await services.GetRequiredService().PopulateProjectDataCache(); return services.GetRequiredService(); } diff --git a/backend/LexData/DataKernel.cs b/backend/LexData/DataKernel.cs index 861252845..e12fc003c 100644 --- a/backend/LexData/DataKernel.cs +++ b/backend/LexData/DataKernel.cs @@ -9,6 +9,7 @@ public static class DataKernel { public static void AddLexData(this IServiceCollection services, bool autoApplyMigrations, + bool useOpenIddict = true, ServiceLifetime dbContextLifeTime = ServiceLifetime.Scoped) { services.AddScoped(); @@ -17,7 +18,7 @@ public static void AddLexData(this IServiceCollection services, options.EnableDetailedErrors(); options.UseNpgsql(serviceProvider.GetRequiredService>().Value.LexBoxConnectionString); options.UseProjectables(); - options.UseOpenIddict(); + if (useOpenIddict) options.UseOpenIddict(); #if DEBUG options.EnableSensitiveDataLogging(); #endif